From deef54fe13c181181b62f8e7e5ec8953e62a6f72 Mon Sep 17 00:00:00 2001 From: czaks Date: Fri, 6 May 2016 18:42:33 +0200 Subject: introduce smart_build_helper --- smart_build.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'smart_build.php') diff --git a/smart_build.php b/smart_build.php index 31d8110a..e287a31a 100644 --- a/smart_build.php +++ b/smart_build.php @@ -1,8 +1,8 @@ bindValue(':id', $thread); if (!$query->execute() || !$query->fetch(PDO::FETCH_ASSOC) ) { - Cache::set("thread_exists_".$b."_".$thread, "no"); + Cache::set("thread_exists_".$b."_".$thread, "no", 3600); return false; } -- cgit v1.3 From a5e22f6d637365042c53f5e9552e8e8444549d6f Mon Sep 17 00:00:00 2001 From: czaks Date: Sun, 8 May 2016 02:50:44 +0200 Subject: split route and controller parts from smart build --- inc/controller.php | 108 +++++++++++++++++++++++++++++++++++++ inc/route.php | 62 ++++++++++++++++++++++ smart_build.php | 153 ++++------------------------------------------------- 3 files changed, 179 insertions(+), 144 deletions(-) create mode 100644 inc/controller.php create mode 100644 inc/route.php (limited to 'smart_build.php') diff --git a/inc/controller.php b/inc/controller.php new file mode 100644 index 00000000..02e33443 --- /dev/null +++ b/inc/controller.php @@ -0,0 +1,108 @@ + $config['max_pages']) return false; + $config['try_smarter'] = true; + $build_pages = array($page); + buildIndex("skip"); + return true; +} + +function sb_api_board($b, $page = 0) { $page = (int)$page; + return sb_board($b, $page + 1); +} + +function sb_thread($b, $thread, $slugcheck = false) { global $config; $thread = (int)$thread; + if ($thread < 1) return false; + + if (!preg_match('/^'.$config['board_regex'].'$/u', $b)) return false; + + if (Cache::get("thread_exists_".$b."_".$thread) == "no") return false; + + $query = prepare(sprintf("SELECT MAX(`id`) AS `max` FROM ``posts_%s``", $b)); + if (!$query->execute()) return false; + + $s = $query->fetch(PDO::FETCH_ASSOC); + $max = $s['max']; + + if ($thread > $max) return false; + + $query = prepare(sprintf("SELECT `id` FROM ``posts_%s`` WHERE `id` = :id AND `thread` IS NULL", $b)); + $query->bindValue(':id', $thread); + + if (!$query->execute() || !$query->fetch(PDO::FETCH_ASSOC) ) { + Cache::set("thread_exists_".$b."_".$thread, "no", 3600); + return false; + } + + if ($slugcheck && $config['slugify']) { + global $request; + + $link = link_for(array("id" => $thread), $slugcheck === 50, array("uri" => $b)); + $link = "/".$b."/".$config['dir']['res'].$link; + + if ($link != $request) { + header("Location: $link", true, 301); + die(); + } + } + + if ($slugcheck == 50) { // Should we really generate +50 page? Maybe there are not enough posts anyway + global $request; + $r = str_replace("+50", "", $request); + $r = substr($r, 1); // Cut the slash + + if (file_exists($r)) return false; + } + + if (!openBoard($b)) return false; + buildThread($thread); + return true; +} + +function sb_thread_slugcheck($b, $thread) { + return sb_thread($b, $thread, true); +} +function sb_thread_slugcheck50($b, $thread) { + return sb_thread($b, $thread, 50); +} + +function sb_api($b) { global $config, $build_pages; + if (!openBoard($b)) return false; + $config['try_smarter'] = true; + $build_pages = array(-1); + buildIndex(); + return true; +} + +function sb_ukko() { + rebuildTheme("ukko", "post-thread"); + return true; +} + +function sb_catalog($b) { + if (!openBoard($b)) return false; + + rebuildTheme("catalog", "post-thread", $b); + return true; +} + +function sb_recent() { + rebuildTheme("recent", "post-thread"); + return true; +} + +function sb_sitemap() { + rebuildTheme("sitemap", "all"); + return true; +} + diff --git a/inc/route.php b/inc/route.php new file mode 100644 index 00000000..66602d77 --- /dev/null +++ b/inc/route.php @@ -0,0 +1,62 @@ + $fun) { + $id = '@^' . preg_quote($id, '@') . '$@u'; + + $id = str_replace('%b', '('.$config['board_regex'].')', $id); + $id = str_replace('%d', '([0-9]+)', $id); + $id = str_replace('%s', '[a-zA-Z0-9-]+', $id); + + $matches = null; + + if (preg_match ($id, $request, $matches)) { + array_shift($matches); + + $reached = array($fun, $matches); + + break; + } + } + + return $reached; +} + diff --git a/smart_build.php b/smart_build.php index e287a31a..58596055 100644 --- a/smart_build.php +++ b/smart_build.php @@ -1,5 +1,7 @@ $config['max_pages']) return false; - $config['try_smarter'] = true; - $build_pages = array($page); - buildIndex("skip"); - return true; -} - -function sb_api_board($b, $page = 0) { $page = (int)$page; - return sb_board($b, $page + 1); -} - -function sb_thread($b, $thread, $slugcheck = false) { global $config; $thread = (int)$thread; - if ($thread < 1) return false; - - if (!preg_match('/^'.$config['board_regex'].'$/u', $b)) return false; - - if (Cache::get("thread_exists_".$b."_".$thread) == "no") return false; - - $query = prepare(sprintf("SELECT MAX(`id`) AS `max` FROM ``posts_%s``", $b)); - if (!$query->execute()) return false; - - $s = $query->fetch(PDO::FETCH_ASSOC); - $max = $s['max']; - - if ($thread > $max) return false; - - $query = prepare(sprintf("SELECT `id` FROM ``posts_%s`` WHERE `id` = :id AND `thread` IS NULL", $b)); - $query->bindValue(':id', $thread); - - if (!$query->execute() || !$query->fetch(PDO::FETCH_ASSOC) ) { - Cache::set("thread_exists_".$b."_".$thread, "no", 3600); - return false; - } - - if ($slugcheck && $config['slugify']) { - global $request; - - $link = link_for(array("id" => $thread), $slugcheck === 50, array("uri" => $b)); - $link = "/".$b."/".$config['dir']['res'].$link; - - if ($link != $request) { - header("Location: $link", true, 301); - die(); - } - } - - if ($slugcheck == 50) { // Should we really generate +50 page? Maybe there are not enough posts anyway - global $request; - $r = str_replace("+50", "", $request); - $r = substr($r, 1); // Cut the slash - - if (file_exists($r)) return false; - } - - if (!openBoard($b)) return false; - buildThread($thread); - return true; -} - -function sb_thread_slugcheck($b, $thread) { - return sb_thread($b, $thread, true); -} -function sb_thread_slugcheck50($b, $thread) { - return sb_thread($b, $thread, 50); -} - -function sb_api($b) { global $config, $build_pages; - if (!openBoard($b)) return false; - $config['try_smarter'] = true; - $build_pages = array(-1); - buildIndex(); - return true; -} - -function sb_ukko() { - rebuildTheme("ukko", "post-thread"); - return true; -} - -function sb_catalog($b) { - if (!openBoard($b)) return false; - - rebuildTheme("catalog", "post-thread", $b); - return true; -} - -function sb_recent() { - rebuildTheme("recent", "post-thread"); - return true; -} - -function sb_sitemap() { - rebuildTheme("sitemap", "all"); - return true; -} - -$entrypoints = array(); - -$entrypoints['/%b/'] = 'sb_board'; -$entrypoints['/%b/'.$config['file_index']] = 'sb_board'; -$entrypoints['/%b/'.$config['file_page']] = 'sb_board'; -$entrypoints['/%b/%d.json'] = 'sb_api_board'; -if ($config['api']['enabled']) { - $entrypoints['/%b/threads.json'] = 'sb_api'; - $entrypoints['/%b/catalog.json'] = 'sb_api'; -} - -$entrypoints['/%b/'.$config['dir']['res'].$config['file_page']] = 'sb_thread_slugcheck'; -$entrypoints['/%b/'.$config['dir']['res'].$config['file_page50']] = 'sb_thread_slugcheck50'; -if ($config['slugify']) { - $entrypoints['/%b/'.$config['dir']['res'].$config['file_page_slug']] = 'sb_thread_slugcheck'; - $entrypoints['/%b/'.$config['dir']['res'].$config['file_page50_slug']] = 'sb_thread_slugcheck50'; -} -if ($config['api']['enabled']) { - $entrypoints['/%b/'.$config['dir']['res'].'%d.json'] = 'sb_thread'; -} - -$entrypoints['/*/'] = 'sb_ukko'; -$entrypoints['/*/index.html'] = 'sb_ukko'; -$entrypoints['/recent.html'] = 'sb_recent'; -$entrypoints['/%b/catalog.html'] = 'sb_catalog'; -$entrypoints['/sitemap.xml'] = 'sb_sitemap'; - -$reached = false; - $request = $_SERVER['REQUEST_URI']; -list($request) = explode('?', $request); - -foreach ($entrypoints as $id => $fun) { - $id = '@^' . preg_quote($id, '@') . '$@u'; - $id = str_replace('%b', '('.$config['board_regex'].')', $id); - $id = str_replace('%d', '([0-9]+)', $id); - $id = str_replace('%s', '[a-zA-Z0-9-]+', $id); +$route = route($request); - $matches = null; - - if (preg_match ($id, $request, $matches)) { - array_shift($matches); - - $reached = call_user_func_array($fun, $matches); - - break; - } +if (!$route) { + $reached = false; +} +else { + list ($fun, $args) = $route; + $reached = call_user_func_array($route); } function die_404() { global $config; -- cgit v1.3 From b6f0317bde1b6d9a9ac5d81a660865a7aa9ad13d Mon Sep 17 00:00:00 2001 From: czaks Date: Sun, 8 May 2016 10:54:30 +0200 Subject: advanced build (1/2): a small refactor of index generating procedure; generation strategies --- inc/config.php | 70 ++++++++++++++++++++++--- inc/functions.php | 102 +++++++++++++++++++++++++++---------- inc/route.php | 5 +- smart_build.php | 9 +++- templates/themes/catalog/theme.php | 13 +++-- templates/themes/recent/theme.php | 5 +- templates/themes/sitemap/theme.php | 6 ++- templates/themes/ukko/theme.php | 6 ++- 8 files changed, 169 insertions(+), 47 deletions(-) (limited to 'smart_build.php') diff --git a/inc/config.php b/inc/config.php index 82fdc882..91804429 100644 --- a/inc/config.php +++ b/inc/config.php @@ -1203,16 +1203,74 @@ // Try not to build pages when we shouldn't have to. $config['try_smarter'] = true; - // EXPERIMENTAL: Defer static HTML building to a moment, when a given file is actually accessed. - // Warning: This option won't run out of the box. You need to tell your webserver, that a file - // for serving 403 and 404 pages is /smart_build.php. Also, you need to turn off indexes. +/* + * ==================== + * Advanced build + * ==================== + */ + + // Strategies for file generation. Also known as an "advanced build". If you don't have performance + // issues, you can safely ignore that part, because it's hard to configure and won't even work on + // your free webhosting. + // + // A strategy is a function, that given the PHP environment and ($fun, $array) variable pair, returns + // an $action array or false. + // + // $fun - a controller function name, see inc/controller.php. This is named after functions, so that + // we can generate the files in daemon. + // + // $array - arguments to be passed + // + // $action - action to be taken. It's an array, and the first element of it is one of the following: + // * "immediate" - generate the page immediately + // * "defer" - defer page generation to a moment a worker daemon gets to build it (serving a stale + // page in the meantime). The remaining arguments are daemon-specific. Daemon isn't + // implemented yet :DDDD inb4 while(true) { generate(Queue::Get()) }; (which is probably it). + // * "build_on_load" - defer page generation to a moment, when the user actually accesses the page. + // This is a smart_build behaviour. You shouldn't use this one too much, if you + // use it for active boards, the server may choke due to a possible race condition. + // See my blog post: https://engine.vichan.net/blog/res/2.html + // + // So, let's assume we want to build a thread 1324 on board /b/, because a new post appeared there. + // We try the first strategy, giving it arguments: 'sb_thread', array('b', 1324). The strategy will + // now return a value $action, denoting an action to do. If $action is false, we try another strategy. + // + // As I said, configuration isn't easy. + // + // 1. chmod 0777 directories: tmp/locks/ and tmp/queue/. + // 2. serve 403 and 404 requests to go thru smart_build.php + // for nginx, this blog post contains config snippets: https://engine.vichan.net/blog/res/2.html + // 3. disable indexes in your webserver + // 4. launch any number of daemons (eg. twice your number of threads?) using the command: + // $ tools/worker.php + // You don't need to do that step if you are not going to use the "defer" option. + // 5. enable smart_build_helper (see below) + // 6. edit the strategies (see inc/functions.php for the builtin ones). You can use lambdas. I will test + // various ones and include one that works best for me. + $config['generation_strategies'] = array(); + // Add a sane strategy. It forces to immediately generate a page user is about to land on. Otherwise, + // it has no opinion, so it needs a fallback strategy. + $config['generation_strategies'][] = 'strategy_sane'; + // Add an immediate catch-all strategy. This is the default function of imageboards: generate all pages + // on post time. + $config['generation_strategies'][] = 'strategy_immediate'; + // NOT RECOMMENDED: Instead of an all-"immediate" strategy, you can use an all-"build_on_load" one (used + // to be initialized using $config['smart_build']; ) for all pages instead of those to be build + // immediately. A rebuild done in this mode should remove all your static files + // $config['generation_strategies'][1] = 'strategy_smart_build'; + + // Deprecated. Leave it false. See above. $config['smart_build'] = false; - // Smart build related: when a file doesn't exist, where should we redirect? + // Use smart_build.php for dispatching missing requests. It may be useful without smart_build or advanced + // build, for example it will regenerate the missing files. + $config['smart_build_helper'] = true; + + // smart_build.php: when a file doesn't exist, where should we redirect? $config['page_404'] = '/404.html'; - // Smart build related: extra entrypoints. - $config['smart_build_entrypoints'] = array(); + // Extra controller entrypoints. Controller is used only by smart_build and advanced build. + $config['controller_entrypoints'] = array(); /* * ==================== diff --git a/inc/functions.php b/inc/functions.php index 1bd1f32f..2957c680 100755 --- a/inc/functions.php +++ b/inc/functions.php @@ -1319,7 +1319,8 @@ function thread_find_page($thread) { return floor(($config['threads_per_page'] + $index) / $config['threads_per_page']); } -function index($page, $mod=false) { +// $brief means that we won't need to generate anything yet +function index($page, $mod=false, $brief = false) { global $board, $config, $debug; $body = ''; @@ -1350,6 +1351,7 @@ function index($page, $mod=false) { unset($cached); } } + if (!isset($cached)) { $posts = prepare(sprintf("SELECT * FROM ``posts_%s`` WHERE `thread` = :id ORDER BY `id` DESC LIMIT :limit", $board['uri'])); $posts->bindValue(':id', $th['id']); @@ -1389,7 +1391,10 @@ function index($page, $mod=false) { } $threads[] = $thread; - $body .= $thread->build(true); + + if (!$brief) { + $body .= $thread->build(true); + } } if ($config['file_board']) { @@ -1610,27 +1615,28 @@ function checkMute() { function buildIndex($global_api = "yes") { global $board, $config, $build_pages; - if (!$config['smart_build']) { - $pages = getPages(); - if (!$config['try_smarter']) - $antibot = create_antibot($board['uri']); + $catalog_api_action = generation_strategy('sb_api', array($board['uri'])); - if ($config['api']['enabled']) { - $api = new Api(); - $catalog = array(); - } + $pages = null; + $antibot = null; + + if ($config['api']['enabled']) { + $api = new Api(); + $catalog = array(); } for ($page = 1; $page <= $config['max_pages']; $page++) { $filename = $board['dir'] . ($page == 1 ? $config['file_index'] : sprintf($config['file_page'], $page)); $jsonFilename = $board['dir'] . ($page - 1) . '.json'; // pages should start from 0 - if ((!$config['api']['enabled'] || $global_api == "skip" || $config['smart_build']) && $config['try_smarter'] - && isset($build_pages) && !empty($build_pages) && !in_array($page, $build_pages) ) + $wont_build_this_page = $config['try_smarter'] && isset($build_pages) && !empty($build_pages) && !in_array($page, $build_pages); + + if ((!$config['api']['enabled'] || $global_api == "skip") && $wont_build_this_page) continue; - if (!$config['smart_build']) { - $content = index($page); + $action = generation_strategy('sb_board', array($board['uri'], $page)); + if ($action == 'rebuild' || $catalog_api_action == 'rebuild') { + $content = index($page, false, $wont_build_this_page); if (!$content) break; @@ -1641,17 +1647,21 @@ function buildIndex($global_api = "yes") { file_write($jsonFilename, $json); $catalog[$page-1] = $threads; - } - if ($config['api']['enabled'] && $global_api != "skip" && $config['try_smarter'] && isset($build_pages) - && !empty($build_pages) && !in_array($page, $build_pages) ) - continue; + if ($wont_build_this_page) continue; + } if ($config['try_smarter']) { $antibot = create_antibot($board['uri'], 0 - $page); $content['current_page'] = $page; } + elseif (!$antibot) { + create_antibot($board['uri']); + } $antibot->reset(); + if (!$pages) { + $pages = getPages(); + } $content['pages'] = $pages; $content['pages'][$page-1]['selected'] = true; $content['btn'] = getPageButtons($content['pages']); @@ -1659,13 +1669,14 @@ function buildIndex($global_api = "yes") { file_write($filename, Element('index.html', $content)); } - else { + elseif ($action == 'delete' || $catalog_api_action == 'delete') { file_unlink($filename); file_unlink($jsonFilename); } } - if (!$config['smart_build'] && $page < $config['max_pages']) { + // $action is an action for our last page + if (($catalog_api_action == 'rebuild' || $action == 'rebuild' || $action == 'delete') && $page < $config['max_pages']) { for (;$page<=$config['max_pages'];$page++) { $filename = $board['dir'] . ($page==1 ? $config['file_index'] : sprintf($config['file_page'], $page)); file_unlink($filename); @@ -1679,13 +1690,13 @@ function buildIndex($global_api = "yes") { // json api catalog if ($config['api']['enabled'] && $global_api != "skip") { - if ($config['smart_build']) { + if ($catalog_api_action == 'delete') { $jsonFilename = $board['dir'] . 'catalog.json'; file_unlink($jsonFilename); $jsonFilename = $board['dir'] . 'threads.json'; file_unlink($jsonFilename); } - else { + elseif ($catalog_api_action == 'rebuild') { $json = json_encode($api->translateCatalog($catalog)); $jsonFilename = $board['dir'] . 'catalog.json'; file_write($jsonFilename, $json); @@ -2204,7 +2215,9 @@ function buildThread($id, $return = false, $mod = false) { if ($config['try_smarter'] && !$mod) $build_pages[] = thread_find_page($id); - if (!$config['smart_build'] || $return || $mod) { + $action = generation_strategy('sb_thread', array($board['uri'], $id)); + + if ($action == 'rebuild' || $return || $mod) { $query = prepare(sprintf("SELECT * FROM ``posts_%s`` WHERE (`thread` IS NULL AND `id` = :id) OR `thread` = :id ORDER BY `thread`,`id`", $board['uri'])); $query->bindValue(':id', $id, PDO::PARAM_INT); $query->execute() or error(db_error($query)); @@ -2239,26 +2252,26 @@ function buildThread($id, $return = false, $mod = false) { )); // json api - if ($config['api']['enabled']) { + if ($config['api']['enabled'] && !$mod) { $api = new Api(); $json = json_encode($api->translateThread($thread)); $jsonFilename = $board['dir'] . $config['dir']['res'] . $id . '.json'; file_write($jsonFilename, $json); } } - else { + elseif($action == 'delete') { $jsonFilename = $board['dir'] . $config['dir']['res'] . $id . '.json'; file_unlink($jsonFilename); } - if ($config['smart_build'] && !$return && !$mod) { + if ($action == 'delete' && !$return && !$mod) { $noko50fn = $board['dir'] . $config['dir']['res'] . link_for(array('id' => $id), true); file_unlink($noko50fn); file_unlink($board['dir'] . $config['dir']['res'] . link_for(array('id' => $id))); - } else if ($return) { + } elseif ($return) { return $body; - } else { + } elseif ($action == 'rebuild') { $noko50fn = $board['dir'] . $config['dir']['res'] . link_for($thread, true); if ($hasnoko50 || file_exists($noko50fn)) { buildThread50($id, $return, $mod, $thread, $antibot); @@ -2788,3 +2801,36 @@ function markdown($s) { return $pd->text($s); } + +function generation_strategy($fun, $array=array()) { global $config; + $action = false; + + foreach ($config['generation_strategies'] as $s) { + if ($strategy = $s($fun, $array)) { + break; + } + } + + switch ($strategy[0]) { + case 'immediate': + return 'rebuild'; + case 'defer': + // Ok, it gets interesting here :) + Queue::add(serialize(array('build', $fun, $array))); + return 'ignore'; + case 'build_on_load': + return 'delete'; + } +} + +function strategy_immediate($fun, $array) { + return array('immediate'); +} + +function strategy_smart_build($fun, $array) { + return array('build_on_load'); +} + +function strategy_sane($fun, $array) { global $config; + return false; +} diff --git a/inc/route.php b/inc/route.php index 66602d77..2a5c1732 100644 --- a/inc/route.php +++ b/inc/route.php @@ -7,7 +7,7 @@ defined('TINYBOARD') or exit; -function route($path) { +function route($path) { global $config; $entrypoints = array(); $entrypoints['/%b/'] = 'sb_board'; @@ -33,8 +33,11 @@ function route($path) { $entrypoints['/*/index.html'] = 'sb_ukko'; $entrypoints['/recent.html'] = 'sb_recent'; $entrypoints['/%b/catalog.html'] = 'sb_catalog'; + $entrypoints['/%b/index.rss'] = 'sb_catalog'; $entrypoints['/sitemap.xml'] = 'sb_sitemap'; + $entrypoints = array_merge($entrypoints, $config['controller_entrypoints']); + $reached = false; list($request) = explode('?', $path); diff --git a/smart_build.php b/smart_build.php index 58596055..7ca5fcbf 100644 --- a/smart_build.php +++ b/smart_build.php @@ -3,14 +3,16 @@ require_once("inc/functions.php"); require_once("inc/route.php"); require_once("inc/controller.php"); -if (!$config['smart_build'] && !$config["smart_build_helper"]) { - die('You need to enable $config["smart_build"] or $config["smart_build_helper"]'); +if (!$config["smart_build_helper"]) { + die('You need to enable $config["smart_build_helper"]'); } $config['smart_build'] = false; // Let's disable it, so we can build the page for real +$config['generation_strategies'] = array('strategy_immediate'); function after_open_board() { global $config; $config['smart_build'] = false; + $config['generation_strategies'] = array('strategy_immediate'); }; $request = $_SERVER['REQUEST_URI']; @@ -59,6 +61,9 @@ if ($reached) { elseif (preg_match('/\.xml$/', $request)) { header("Content-Type", "application/xml"); } + elseif (preg_match('/\.rss$/', $request)) { + header("Content-Type", "application/rss+xml"); + } else { header("Content-Type", "text/html; charset=utf-8"); } diff --git a/templates/themes/catalog/theme.php b/templates/themes/catalog/theme.php index 239d4dff..4f512c03 100644 --- a/templates/themes/catalog/theme.php +++ b/templates/themes/catalog/theme.php @@ -16,20 +16,25 @@ if ($action == 'all') { foreach ($boards as $board) { $b = new Catalog(); - if ($config['smart_build']) { + + $action = generation_strategy("sb_catalog", array($board)); + if ($action == 'delete') { file_unlink($config['dir']['home'] . $board . '/catalog.html'); + file_unlink($config['dir']['home'] . $board . '/index.rss'); } - else { + elseif ($action == 'rebuild') { $b->build($settings, $board); } } } elseif ($action == 'post-thread' || ($settings['update_on_posts'] && $action == 'post') || ($settings['update_on_posts'] && $action == 'post-delete') && in_array($board, $boards)) { $b = new Catalog(); - if ($config['smart_build']) { + $action = generation_strategy("sb_catalog", array($board)); + if ($action == 'delete') { file_unlink($config['dir']['home'] . $board . '/catalog.html'); + file_unlink($config['dir']['home'] . $board . '/index.rss'); } - else { + elseif ($action == 'rebuild') { $b->build($settings, $board); } } diff --git a/templates/themes/recent/theme.php b/templates/themes/recent/theme.php index f44e2529..95921c35 100644 --- a/templates/themes/recent/theme.php +++ b/templates/themes/recent/theme.php @@ -25,10 +25,11 @@ $this->excluded = explode(' ', $settings['exclude']); if ($action == 'all' || $action == 'post' || $action == 'post-thread' || $action == 'post-delete') { - if ($config['smart_build']) { + $action = generation_strategy('sb_recent', array()); + if ($action == 'delete') { file_unlink($config['dir']['home'] . $settings['html']); } - else { + elseif ($action == 'rebuild') { file_write($config['dir']['home'] . $settings['html'], $this->homepage($settings)); } } diff --git a/templates/themes/sitemap/theme.php b/templates/themes/sitemap/theme.php index 52779d53..6bc035bb 100644 --- a/templates/themes/sitemap/theme.php +++ b/templates/themes/sitemap/theme.php @@ -23,10 +23,12 @@ } } - if ($config['smart_build']) { + $action = generation_strategy('sb_sitemap', array()); + + if ($action == 'delete') { file_unlink($settings['path']); } - else { + elseif ($action == 'rebuild') { $boards = explode(' ', $settings['boards']); $threads = array(); diff --git a/templates/themes/ukko/theme.php b/templates/themes/ukko/theme.php index d6fc303c..e572c467 100644 --- a/templates/themes/ukko/theme.php +++ b/templates/themes/ukko/theme.php @@ -11,10 +11,12 @@ return; } - if ($config['smart_build']) { + $action = generation_strategy('sb_ukko', array()); + + if ($action == 'delete') { file_unlink($settings['uri'] . '/index.html'); } - else { + elseif ($action == 'rebuild') { file_write($settings['uri'] . '/index.html', $ukko->build()); } } -- cgit v1.3 From 12e6aba5d4a2965591b3df76e4f6c19e35475700 Mon Sep 17 00:00:00 2001 From: czaks Date: Sun, 8 May 2016 13:20:00 +0200 Subject: (2/2) advanced build. implement a daemon that will build static pages. implement a queue and a lock. fix notice in bans. and it even works! the daemon is basic right now, it could work in a mode that it will defer building certain pages until a certain time. --- inc/bans.php | 2 +- inc/config.php | 9 ++++++++- inc/functions.php | 40 +++++++++++++++++++++++++++++++++++----- smart_build.php | 2 +- tmp/queue/generate/.gitkeep | 0 tools/worker.php | 31 +++++++++++++++++++++++++++++++ 6 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 tmp/queue/generate/.gitkeep create mode 100755 tools/worker.php (limited to 'smart_build.php') diff --git a/inc/bans.php b/inc/bans.php index 87e06e28..c468eb64 100644 --- a/inc/bans.php +++ b/inc/bans.php @@ -166,7 +166,7 @@ class Bans { if ($ban['post']) { $post = json_decode($ban['post']); - $ban['message'] = $post->body; + $ban['message'] = isset($post->body) ? $post->body : 0; } unset($ban['ipstart'], $ban['ipend'], $ban['post'], $ban['creator']); diff --git a/inc/config.php b/inc/config.php index 91804429..5926eb1d 100644 --- a/inc/config.php +++ b/inc/config.php @@ -103,7 +103,7 @@ /* * ==================== - * Cache settings + * Cache, lock and queue settings * ==================== */ @@ -120,6 +120,7 @@ // $config['cache']['enabled'] = 'apc'; // $config['cache']['enabled'] = 'memcached'; // $config['cache']['enabled'] = 'redis'; + // $config['cache']['enabled'] = 'fs'; // Timeout for cached objects such as posts and HTML. $config['cache']['timeout'] = 60 * 60 * 48; // 48 hours @@ -142,6 +143,12 @@ // (this file will be explicitly loaded during cache hit, but not during cache miss). $config['cache_config'] = false; + // Define a lock driver. + $config['lock']['enabled'] = 'fs'; + + // Define a queue driver. + $config['queue']['enabled'] = 'fs'; // xD + /* * ==================== * Cookie settings diff --git a/inc/functions.php b/inc/functions.php index 8329e774..0cfdb72f 100755 --- a/inc/functions.php +++ b/inc/functions.php @@ -19,6 +19,8 @@ require_once 'inc/database.php'; require_once 'inc/events.php'; require_once 'inc/api.php'; require_once 'inc/mod/auth.php'; +require_once 'inc/lock.php'; +require_once 'inc/queue.php'; require_once 'inc/polyfill.php'; @include_once 'inc/lib/parsedown/Parsedown.php'; // fail silently, this isn't a critical piece of code @@ -93,6 +95,8 @@ function loadConfig() { 'db', 'api', 'cache', + 'lock', + 'queue', 'cookies', 'error', 'dir', @@ -1749,7 +1753,6 @@ function buildJavascript() { function checkDNSBL() { global $config; - if (isIPv6()) return; // No IPv6 support yet. @@ -2806,17 +2809,17 @@ function generation_strategy($fun, $array=array()) { global $config; $action = false; foreach ($config['generation_strategies'] as $s) { - if ($strategy = $s($fun, $array)) { + if ($action = $s($fun, $array)) { break; } } - switch ($strategy[0]) { + switch ($action[0]) { case 'immediate': return 'rebuild'; case 'defer': // Ok, it gets interesting here :) - Queue::add(serialize(array('build', $fun, $array))); + get_queue('generate')->push(serialize(array('build', $fun, $array, $action))); return 'ignore'; case 'build_on_load': return 'delete'; @@ -2832,5 +2835,32 @@ function strategy_smart_build($fun, $array) { } function strategy_sane($fun, $array) { global $config; - return false; + // Well, ideally a sane strategy would involve a more stringent checking, + // but let's at least have something to get the ball rolling :^) + + if (php_sapi_name() == 'cli') return false; + else if (isset($_POST['mod']) || isset($_POST['json_response'])) return false; + else if ($fun == 'sb_thread' || ($fun == 'sb_board' && $array[1] == 1)) return array('immediate'); + else return false; +} + +// My first, test strategy. +function strategy_first($fun, $array) { + switch ($fun) { + case 'sb_thread': + return array('defer'); + case 'sb_board': + if ($array[1] > 8) return array('build_on_load'); + else return array('defer'); + case 'sb_api': + return array('defer'); + case 'sb_catalog': + return array('defer'); + case 'sb_recent': + return array('build_on_load'); + case 'sb_sitemap': + return array('build_on_load'); + case 'sb_ukko': + return array('defer'); + } } diff --git a/smart_build.php b/smart_build.php index 7ca5fcbf..cfac446a 100644 --- a/smart_build.php +++ b/smart_build.php @@ -24,7 +24,7 @@ if (!$route) { } else { list ($fun, $args) = $route; - $reached = call_user_func_array($route); + $reached = call_user_func_array($fun, $args); } function die_404() { global $config; diff --git a/tmp/queue/generate/.gitkeep b/tmp/queue/generate/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tools/worker.php b/tools/worker.php new file mode 100755 index 00000000..e19fe1c6 --- /dev/null +++ b/tools/worker.php @@ -0,0 +1,31 @@ +#!/usr/bin/php +pop(2); + foreach ($q as $v) { + list($__, $func, $ary, $action) = unserialize($v); + echo "Starting to generate $func ".implode(" ", $ary)."... "; + + call_user_func_array($func, $ary); + + echo "done!\n"; + } + if (!$q) usleep(20000); // 0.02s +} -- cgit v1.3