From 577a8b991d10b898e0025f5f45bb38cafff5080e Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Thu, 29 Aug 2013 12:52:31 +1000 Subject: Performance: Use only one query for validating >>X links in posts --- inc/functions.php | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) (limited to 'inc/functions.php') diff --git a/inc/functions.php b/inc/functions.php index 1bd185a9..321c65d0 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -1576,29 +1576,41 @@ function markup(&$body, $track_cites = false) { $skip_chars = 0; $body_tmp = $body; - + + $search_cites = array(); + foreach ($cites as $matches) { + $search_cites[] = '`id` = ' . $matches[2][0]; + } + $search_cites = array_unique($search_cites); + + $query = query(sprintf('SELECT `thread`, `id` FROM ``posts_%s`` WHERE ' . + implode(' OR ', $search_cites), $board['uri'])) or error(db_error()); + + $cited_posts = array(); + while ($cited = $query->fetch(PDO::FETCH_ASSOC)) { + $cited_posts[$cited['id']] = $cited['thread'] ? $cited['thread'] : false; + } + foreach ($cites as $matches) { $cite = $matches[2][0]; - $query = prepare(sprintf("SELECT `thread`,`id` FROM ``posts_%s`` WHERE `id` = :id LIMIT 1", $board['uri'])); - $query->bindValue(':id', $cite); - $query->execute() or error(db_error($query)); // preg_match_all is not multibyte-safe foreach ($matches as &$match) { $match[1] = mb_strlen(substr($body_tmp, 0, $match[1])); } - if ($post = $query->fetch(PDO::FETCH_ASSOC)) { + if (isset($cited_posts[$cite])) { $replacement = '' . - '>>' . $cite . - ''; + $config['root'] . $board['dir'] . $config['dir']['res'] . + ($cited_posts[$cite] ? $cited_posts[$cite] : $cite) . '.html#' . $cite . '">' . + '>>' . $cite . + ''; $body = mb_substr_replace($body, $matches[1][0] . $replacement . $matches[3][0], $matches[0][1] + $skip_chars, mb_strlen($matches[0][0])); $skip_chars += mb_strlen($matches[1][0] . $replacement . $matches[3][0]) - mb_strlen($matches[0][0]); if ($track_cites && $config['track_cites']) - $tracked_cites[] = array($board['uri'], $post['id']); + $tracked_cites[] = array($board['uri'], $cite); } } } @@ -1612,6 +1624,8 @@ function markup(&$body, $track_cites = false) { $skip_chars = 0; $body_tmp = $body; + // TODO: Make this use fewer queries, similar to local board cites. + foreach ($cites as $matches) { $_board = $matches[2][0]; $cite = @$matches[3][0]; -- cgit v1.3 From 3545e2406edc441123d0796ffff12fc871bf7ec1 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Thu, 29 Aug 2013 13:31:02 +1000 Subject: Perforamnce: Much more efficient >>X and >>>/board/X --- inc/functions.php | 111 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 82 insertions(+), 29 deletions(-) (limited to 'inc/functions.php') diff --git a/inc/functions.php b/inc/functions.php index 321c65d0..f260a0ce 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -1623,8 +1623,66 @@ function markup(&$body, $track_cites = false) { $skip_chars = 0; $body_tmp = $body; - - // TODO: Make this use fewer queries, similar to local board cites. + + if (isset($cited_posts)) { + // Carry found posts from local board >>X links + foreach ($cited_posts as $cite => $thread) { + $cited_posts[$cite] = $config['root'] . $board['dir'] . $config['dir']['res'] . + ($thread ? $thread : $cite) . '.html#' . $cite; + } + + $cited_posts = array( + $board['uri'] => $cited_posts + ); + } else + $cited_posts = array(); + + $crossboard_indexes = array(); + $search_cites_boards = array(); + + foreach ($cites as $matches) { + $_board = $matches[2][0]; + $cite = @$matches[3][0]; + + if (!isset($search_cites_boards[$_board])) + $search_cites_boards[$_board] = array(); + $search_cites_boards[$_board][] = $cite; + } + + $tmp_board = $board['uri']; + + foreach ($search_cites_boards as $_board => $search_cites) { + $clauses = array(); + foreach ($search_cites as $cite) { + if (!$cite || isset($cited_posts[$_board][$cite])) + continue; + $clauses[] = '`id` = ' . $cite; + } + $clauses = array_unique($clauses); + + if ($board['uri'] != $_board) { + if (!openBoard($_board)) + continue; // Unknown board + } + + if (!empty($clauses)) { + $cited_posts[$_board] = array(); + + $query = query(sprintf('SELECT `thread`, `id` FROM ``posts_%s`` WHERE ' . + implode(' OR ', $clauses), $board['uri'])) or error(db_error()); + + while ($cite = $query->fetch(PDO::FETCH_ASSOC)) { + $cited_posts[$_board][$cite['id']] = $config['root'] . $board['dir'] . $config['dir']['res'] . + ($cite['thread'] ? $cite['thread'] : $cite['id']) . '.html#' . $cite['id']; + } + } + + $crossboard_indexes[$_board] = $config['root'] . $board['dir'] . $config['file_index']; + } + + // Restore old board + if ($board['uri'] != $tmp_board) + openBoard($tmp_board); foreach ($cites as $matches) { $_board = $matches[2][0]; @@ -1638,39 +1696,34 @@ function markup(&$body, $track_cites = false) { // Temporarily store board information because it will be overwritten $tmp_board = $board['uri']; - // Check if the board exists, and load settings - if (openBoard($_board)) { - if ($cite) { - $query = prepare(sprintf("SELECT `thread`,`id` FROM ``posts_%s`` WHERE `id` = :id LIMIT 1", $board['uri'])); - $query->bindValue(':id', $cite); - $query->execute() or error(db_error($query)); - - if ($post = $query->fetch(PDO::FETCH_ASSOC)) { - $replacement = '' . - '>>>/' . $_board . '/' . $cite . - ''; - - $body = mb_substr_replace($body, $matches[1][0] . $replacement . $matches[4][0], $matches[0][1] + $skip_chars, mb_strlen($matches[0][0])); - $skip_chars += mb_strlen($matches[1][0] . $replacement . $matches[4][0]) - mb_strlen($matches[0][0]); - - if ($track_cites && $config['track_cites']) - $tracked_cites[] = array($board['uri'], $post['id']); - } - } else { - $replacement = '' . - '>>>/' . $_board . '/' . - ''; + if ($cite) { + if (isset($cited_posts[$_board][$cite])) { + $link = $cited_posts[$_board][$cite]; + + $replacement = '' . + '>>>/' . $_board . '/' . $cite . + ''; + $body = mb_substr_replace($body, $matches[1][0] . $replacement . $matches[4][0], $matches[0][1] + $skip_chars, mb_strlen($matches[0][0])); $skip_chars += mb_strlen($matches[1][0] . $replacement . $matches[4][0]) - mb_strlen($matches[0][0]); + + if ($track_cites && $config['track_cites']) + $tracked_cites[] = array($_board, $cite); } + } elseif(isset($crossboard_indexes[$_board])) { + $replacement = '' . + '>>>/' . $_board . '/' . + ''; + $body = mb_substr_replace($body, $matches[1][0] . $replacement . $matches[4][0], $matches[0][1] + $skip_chars, mb_strlen($matches[0][0])); + $skip_chars += mb_strlen($matches[1][0] . $replacement . $matches[4][0]) - mb_strlen($matches[0][0]); } - - // Restore main board settings - openBoard($tmp_board); } } + + $tracked_cites = array_unique($tracked_cites, SORT_REGULAR); $body = preg_replace("/^\s*>.*$/m", '$0', $body); -- cgit v1.3 From 334e69b69f2c0191bfcfb9ff1ef3d7d8ffeb8a9e Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Thu, 29 Aug 2013 13:32:55 +1000 Subject: Remove old code --- inc/functions.php | 3 --- 1 file changed, 3 deletions(-) (limited to 'inc/functions.php') diff --git a/inc/functions.php b/inc/functions.php index f260a0ce..d4a5766a 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -1693,9 +1693,6 @@ function markup(&$body, $track_cites = false) { $match[1] = mb_strlen(substr($body_tmp, 0, $match[1])); } - // Temporarily store board information because it will be overwritten - $tmp_board = $board['uri']; - if ($cite) { if (isset($cited_posts[$_board][$cite])) { $link = $cited_posts[$_board][$cite]; -- cgit v1.3 From 7b817eea1119a9bd243497d68ba27059533ba5d8 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Thu, 29 Aug 2013 15:56:36 +1000 Subject: Fix markup again. And add the option to repair fucked up nesting (and more) with HTML Tidy ($config['markup_repair_tidy']) --- inc/config.php | 12 ++++++++---- inc/functions.php | 11 ++++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) (limited to 'inc/functions.php') diff --git a/inc/config.php b/inc/config.php index 9f68d04b..f9dc9912 100644 --- a/inc/config.php +++ b/inc/config.php @@ -459,10 +459,10 @@ */ // "Wiki" markup syntax ($config['wiki_markup'] in pervious versions): - $config['markup'][] = array("/'''([^<]+?)'''/", "\$1"); - $config['markup'][] = array("/''([^<]+?)''/", "\$1"); - $config['markup'][] = array("/\*\*([^<]+?)\*\*/", "\$1"); - // $config['markup'][] = array("/^[ |\t]*==([^<]+?)==[ |\t]*$/m", "\$1"); + $config['markup'][] = array("/'''(.+?)'''/", "\$1"); + $config['markup'][] = array("/''(.+?)''/", "\$1"); + $config['markup'][] = array("/\*\*(.+?)\*\*/", "\$1"); + // $config['markup'][] = array("/^[ |\t]*==(.+?)==[ |\t]*$/m", "\$1"); // Highlight PHP code wrapped in tags (PHP 5.3+) // $config['markup'][] = array( @@ -472,6 +472,10 @@ // } // ); + // Repair markup with HTML Tidy. This may be slower, but most if the time it solves nesting mistakes. + // Tinyboad, at the time of writing this, can not prevent out-of-order markup tags (eg. "**''test**''). + $config['markup_repair_tidy'] = false; + // Always regenerate markup. This isn't recommended and should only be used for debugging; by default, // Tinyboard only parses post markup when it needs to, and keeps post-markup HTML in the database. This // will significantly impact performance when enabled. diff --git a/inc/functions.php b/inc/functions.php index d4a5766a..d006d0ad 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -1728,7 +1728,16 @@ function markup(&$body, $track_cites = false) { $body = preg_replace('/\s+$/', '', $body); $body = preg_replace("/\n/", '
', $body); - + + if ($config['markup_repair_tidy']) { + $tidy = new tidy(); + $body = $tidy->repairString($body, array( + 'doctype' => 'omit' + )); + $body = str_replace("\n", '', $body); + $body = preg_replace('@^.+|.+$@', '', $body); + } + return $tracked_cites; } -- cgit v1.3 From a8e37543757a6f5e6e841a3745cdb10b2247a269 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Thu, 29 Aug 2013 18:55:25 +1000 Subject: Minor $config['try_smarter'] work --- inc/functions.php | 5 ++++- inc/template.php | 4 +++- post.php | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) (limited to 'inc/functions.php') diff --git a/inc/functions.php b/inc/functions.php index d006d0ad..8b338063 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -1306,7 +1306,7 @@ function buildIndex() { for ($page = 1; $page <= $config['max_pages']; $page++) { $filename = $board['dir'] . ($page == 1 ? $config['file_index'] : sprintf($config['file_page'], $page)); - if ($config['try_smarter'] && isset($build_pages) && count($build_pages) + if ($config['try_smarter'] && isset($build_pages) && !empty($build_pages) && !in_array($page, $build_pages) && is_file($filename)) continue; $content = index($page); @@ -1352,6 +1352,9 @@ function buildIndex() { $jsonFilename = $board['dir'] . 'catalog.json'; file_write($jsonFilename, $json); } + + if ($config['try_smarter']) + $build_pages = array(); } function buildJavascript() { diff --git a/inc/template.php b/inc/template.php index 6f85e78b..6d417631 100644 --- a/inc/template.php +++ b/inc/template.php @@ -35,7 +35,7 @@ function load_twig() { } function Element($templateFile, array $options) { - global $config, $debug, $twig; + global $config, $debug, $twig, $build_pages; if (!$twig) load_twig(); @@ -51,6 +51,8 @@ function Element($templateFile, array $options) { unset($debug['start']); unset($debug['start_debug']); } + if ($config['try_smarter'] && isset($build_pages) && !empty($build_pages)) + $debug['build_pages'] = $build_pages; $debug['included'] = get_included_files(); $debug['memory'] = round(memory_get_usage(true) / (1024 * 1024), 2) . ' MiB'; $options['body'] .= diff --git a/post.php b/post.php index 7ca9c1a9..de4bc07d 100644 --- a/post.php +++ b/post.php @@ -682,7 +682,7 @@ if (isset($_POST['delete'])) { incrementSpamHash($post['antispam_hash']); } - if (isset($post['tracked_cites']) && count($post['tracked_cites'])) { + if (isset($post['tracked_cites']) && !empty($post['tracked_cites'])) { $insert_rows = array(); foreach ($post['tracked_cites'] as $cite) { $insert_rows[] = '(' . -- cgit v1.3 From 8144e517e9f3913ef024a45d702f0185a825df39 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Thu, 29 Aug 2013 20:40:39 +1000 Subject: Don't set arbitrary cache timeouts for DNS stuff --- inc/functions.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'inc/functions.php') diff --git a/inc/functions.php b/inc/functions.php index 8b338063..f99713f7 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -2003,7 +2003,7 @@ function rDNS($ip_addr) { } if ($config['cache']['enabled']) - cache::set('rdns_' . $ip_addr, $host, 3600); + cache::set('rdns_' . $ip_addr, $host); return $host; } @@ -2028,7 +2028,7 @@ function DNS($host) { } if ($config['cache']['enabled']) - cache::set('dns_' . $host, $ip_addr !== false ? $ip_addr : '?', 3600); + cache::set('dns_' . $host, $ip_addr !== false ? $ip_addr : '?'); return $ip_addr; } -- cgit v1.3 From 471525a58bc59399066eaffb6e593b432ec854a8 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Thu, 29 Aug 2013 21:05:03 +1000 Subject: HTML Tidy fixes: UTF-8, preserving whitespace, keep attributes literal, don't wrap text --- inc/functions.php | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'inc/functions.php') diff --git a/inc/functions.php b/inc/functions.php index f99713f7..22a6d9ff 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -1731,16 +1731,22 @@ function markup(&$body, $track_cites = false) { $body = preg_replace('/\s+$/', '', $body); $body = preg_replace("/\n/", '
', $body); - + if ($config['markup_repair_tidy']) { $tidy = new tidy(); - $body = $tidy->repairString($body, array( - 'doctype' => 'omit' - )); + $body = $tidy->repairString(str_replace(' ', ' ', $body), array( + 'doctype' => 'omit', + 'bare' => true, + 'literal-attributes' => true, + 'quote-nbsp' => true, + 'indent' => false, + 'show-body-only' => true, + 'wrap' => 0, + 'output-bom' => false, + ), 'utf8'); $body = str_replace("\n", '', $body); - $body = preg_replace('@^.+|.+$@', '', $body); } - + return $tracked_cites; } -- cgit v1.3 From 738179c766953fa45670e451875210c6874a41f2 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Thu, 29 Aug 2013 21:24:38 +1000 Subject: HTML Tidy bug fix: broken attributes, and preserve whitespace properly --- inc/functions.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'inc/functions.php') diff --git a/inc/functions.php b/inc/functions.php index 22a6d9ff..e634b87e 100644 --- a/inc/functions.php +++ b/inc/functions.php @@ -1555,6 +1555,9 @@ function markup(&$body, $track_cites = false) { if ($num_links > $config['max_links']) error($config['error']['toomanylinks']); } + + if ($config['markup_repair_tidy']) + $body = str_replace(' ', '  ', $body); if ($config['auto_unicode']) { $body = unicodify($body); @@ -1731,18 +1734,21 @@ function markup(&$body, $track_cites = false) { $body = preg_replace('/\s+$/', '', $body); $body = preg_replace("/\n/", '
', $body); - + if ($config['markup_repair_tidy']) { $tidy = new tidy(); - $body = $tidy->repairString(str_replace(' ', ' ', $body), array( + $body = $tidy->repairString($body, array( 'doctype' => 'omit', 'bare' => true, 'literal-attributes' => true, - 'quote-nbsp' => true, 'indent' => false, 'show-body-only' => true, 'wrap' => 0, 'output-bom' => false, + 'output-html' => true, + 'newline' => 'LF', + 'quiet' => true, + ), 'utf8'); $body = str_replace("\n", '', $body); } -- cgit v1.3