From f309e4037c11197e4392e402992882358e4678c7 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Fri, 6 Sep 2013 23:09:18 +1000 Subject: Better and faster basic flood prevention, while merging it into $config['filters']. --- inc/config.php | 177 +++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 114 insertions(+), 63 deletions(-) (limited to 'inc/config.php') diff --git a/inc/config.php b/inc/config.php index e8b65967..81cae233 100644 --- a/inc/config.php +++ b/inc/config.php @@ -168,13 +168,6 @@ * ==================== */ - // Minimum time between between each post by the same IP address. - $config['flood_time'] = 10; - // Minimum time between between each post with the exact same content AND same IP address. - $config['flood_time_ip'] = 120; - // Same as above but by a different IP address. (Same content, not necessarily same IP address.) - $config['flood_time_same'] = 30; - /* * To further prevent spam and abuse, you can use DNS blacklists (DNSBL). A DNSBL is a list of IP * addresses published through the Internet Domain Name Service (DNS) either as a zone file that can be @@ -283,6 +276,118 @@ $config['recaptcha_public'] = '6LcXTcUSAAAAAKBxyFWIt2SO8jwx4W7wcSMRoN3f'; $config['recaptcha_private'] = '6LcXTcUSAAAAAOGVbVdhmEM1_SyRF4xTKe8jbzf_'; + /* + * Custom filters detect certain posts and reject/ban accordingly. They are made up of a condition and an + * action (for when ALL conditions are met). As every single post has to be put through each filter, + * having hundreds probably isn't ideal as it could slow things down. + * + * By default, the custom filters array is populated with basic flood prevention conditions. This + * includes forcing users to wait at least 5 seconds between posts. To disable (or amend) these flood + * prevention settings, you will need to empty the $config['filters'] array first. You can do so by + * adding "$config['filters'] = array();" to inc/instance-config.php. Basic flood prevention used to be + * controlled solely by config variables such as $config['flood_time'] and $config['flood_time_ip'], and + * it still is, as long as you leave the relevant $config['filters'] intact. These old config variables + * still exist for backwards-compatability and general convenience. + * + * Read more: http://tinyboard.org/docs/index.php?p=Config/Filters + */ + + // Minimum time between between each post by the same IP address. + $config['flood_time'] = 10; + // Minimum time between between each post with the exact same content AND same IP address. + $config['flood_time_ip'] = 120; + // Same as above but by a different IP address. (Same content, not necessarily same IP address.) + $config['flood_time_same'] = 30; + + // Minimum time between posts by the same IP address (all boards). + $config['filters'][] = array( + 'condition' => array( + 'flood-match' => array('ip'), // Only match IP address + 'flood-time' => &$config['flood_time'] // 10 seconds minimum + ), + 'action' => 'reject', + 'message' => &$config['error']['flood'] + ); + + // Minimum time between posts by the same IP address with the same text. + $config['filters'][] = array( + 'condition' => array( + 'flood-match' => array('ip', 'body'), // Match IP address and post body + 'flood-time' => &$config['flood_time_ip'] // 2 minutes minimum + ), + 'action' => 'reject', + 'message' => &$config['error']['flood'] + ); + + // Minimum time between posts with the same text. (Same content, but not always the same IP address.) + $config['filters'][] = array( + 'condition' => array( + 'flood-match' => array('body'), // Match IP address and post body + 'flood-time' => &$config['flood_time_same'] // 30 seconds minimum + ), + 'action' => 'reject', + 'message' => &$config['error']['flood'] + ); + + // Example: Minimum time between posts with the same file hash. + // $config['filters'][] = array( + // 'condition' => array( + // 'flood-match' => array('file'), // Match file hash + // 'flood-time' => 60 * 2 // 2 minutes minimum + // ), + // 'action' => 'reject', + // 'message' => &$config['error']['flood'] + // ); + + // An example of blocking an imaginary known spammer, who keeps posting a reply with the name "surgeon", + // ending his posts with "regards, the surgeon" or similar. + // $config['filters'][] = array( + // 'condition' => array( + // 'name' => '/^surgeon$/', + // 'body' => '/regards,\s+(the )?surgeon$/i', + // 'OP' => false + // ), + // 'action' => 'reject', + // 'message' => 'Go away, spammer.' + // ); + + // Same as above, but issuing a 3-hour ban instead of just reject the post. + // $config['filters'][] = array( + // 'condition' => array( + // 'name' => '/^surgeon$/', + // 'body' => '/regards,\s+(the )?surgeon$/i', + // 'OP' => false + // ), + // 'action' => 'ban', + // 'expires' => 60 * 60 * 3, // 3 hours + // 'reason' => 'Go away, spammer.' + // ); + + // PHP 5.3+ (anonymous functions) + // There is also a "custom" condition, making the possibilities of this feature pretty much endless. + // This is a bad example, because there is already a "name" condition built-in. + // $config['filters'][] = array( + // 'condition' => array( + // 'body' => '/h$/i', + // 'OP' => false, + // 'custom' => function($post) { + // if($post['name'] == 'Anonymous') + // return true; + // else + // return false; + // } + // ), + // 'action' => 'reject' + // ); + + // Filter flood prevention conditions ("flood-match") depend on a table which contains a cache of recent + // posts across all boards. This table is automatically purged of older posts, determining the maximum + // "age" by looking at each filter. However, when determining the maximum age, Tinyboard does not look + // outside the current board. This means that if you have a special flood condition for a specific board + // (contained in a board configuration file) which has a flood-time greater than any of those in the + // global configuration, you need to set the following variable to the maximum flood-time condition value. + // $config['flood_cache'] = 60 * 60 * 24; // 24 hours + /* * ==================== * Post settings @@ -401,57 +506,6 @@ // Require users to see the ban page at least once for a ban even if it has since expired. $config['require_ban_view'] = true; - /* - * Custom filters detect certain posts and reject/ban accordingly. They are made up of a - * condition and an action (for when ALL conditions are met). As every single post has to - * be put through each filter, having hundreds probably isn’t ideal as it could slow things down. - * - * Read more: http://tinyboard.org/docs/index.php?p=Config/Filters - * - * This used to be named $config['flood_filters'] (still exists as an alias). - */ - - // An example of blocking an imaginary known spammer, who keeps posting a reply with the name "surgeon", - // ending his posts with "regards, the surgeon" or similar. - // $config['filters'][] = array( - // 'condition' => array( - // 'name' => '/^surgeon$/', - // 'body' => '/regards,\s+(the )?surgeon$/i', - // 'OP' => false - // ), - // 'action' => 'reject', - // 'message' => 'Go away, spammer.' - // ); - - // Same as above, but issuing a 3-hour ban instead of just reject the post. - // $config['filters'][] = array( - // 'condition' => array( - // 'name' => '/^surgeon$/', - // 'body' => '/regards,\s+(the )?surgeon$/i', - // 'OP' => false - // ), - // 'action' => 'ban', - // 'expires' => 60 * 60 * 3, // 3 hours - // 'reason' => 'Go away, spammer.' - // ); - - // PHP 5.3+ (anonymous functions) - // There is also a "custom" condition, making the possibilities of this feature pretty much endless. - // This is a bad example, because there is already a "name" condition built-in. - // $config['filters'][] = array( - // 'condition' => array( - // 'body' => '/h$/i', - // 'OP' => false, - // 'custom' => function($post) { - // if($post['name'] == 'Anonymous') - // return true; - // else - // return false; - // } - // ), - // 'action' => 'reject' - // ); - /* * ==================== * Markup settings @@ -593,10 +647,6 @@ // that as a thumbnail instead of resizing/redrawing. $config['minimum_copy_resize'] = false; - // Image hashing function. There's really no reason to change this. - // sha1_file, md5_file, etc. You can also define your own similar function. - $config['file_hash'] = 'sha1_file'; - // Maximum image upload size in bytes. $config['max_filesize'] = 10 * 1024 * 1024; // 10MB // Maximum image dimensions. @@ -1156,7 +1206,8 @@ // Post bypass unoriginal content check on robot-enabled boards $config['mod']['postunoriginal'] = ADMIN; // Bypass flood check - $config['mod']['flood'] = ADMIN; + $config['mod']['bypass_filters'] = ADMIN; + $config['mod']['flood'] = &$config['mod']['bypass_filters']; // Raw HTML posting $config['mod']['rawhtml'] = ADMIN; -- cgit v1.3 From e9ccc5d72d121859c8a766bd64a87d45d42f62a2 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sat, 7 Sep 2013 12:40:35 +1000 Subject: Optionally EXPLAIN all SQL queries when in debug mode --- inc/config.php | 2 ++ inc/database.php | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) (limited to 'inc/config.php') diff --git a/inc/config.php b/inc/config.php index 81cae233..43550dc3 100644 --- a/inc/config.php +++ b/inc/config.php @@ -44,6 +44,8 @@ $config['debug'] = false; // For development purposes. Displays (and "dies" on) all errors and warnings. Turn on with the above. $config['verbose_errors'] = true; + // EXPLAIN all SQL queries (when in debug mode). + $config['debug_explain'] = false; // Directory where temporary files will be created. $config['tmp'] = sys_get_temp_dir(); diff --git a/inc/database.php b/inc/database.php index d2f1af04..2c60015a 100644 --- a/inc/database.php +++ b/inc/database.php @@ -7,21 +7,29 @@ defined('TINYBOARD') or exit; class PreparedQueryDebug { - protected $query; + protected $query, $explain_query = false; public function __construct($query) { - global $pdo; + global $pdo, $config; $query = preg_replace("/[\n\t]+/", ' ', $query); $this->query = $pdo->prepare($query); + if ($config['debug'] && $config['debug_explain'] && preg_match('/^(SELECT|INSERT|UPDATE|DELETE) /', $query)) + $this->explain_query = $pdo->prepare("EXPLAIN $query"); } public function __call($function, $args) { global $config, $debug; if ($config['debug'] && $function == 'execute') { + if ($this->explain_query) { + $this->explain_query->execute() or error(db_error($explain_query)); + } $start = microtime(true); } + if ($this->explain_query && $function == 'bindValue') + call_user_func_array(array($this->explain_query, $function), $args); + $return = call_user_func_array(array($this->query, $function), $args); if ($config['debug'] && $function == 'execute') { @@ -29,6 +37,7 @@ class PreparedQueryDebug { $debug['sql'][] = array( 'query' => $this->query->queryString, 'rows' => $this->query->rowCount(), + 'explain' => $this->explain_query ? $this->explain_query->fetchAll(PDO::FETCH_ASSOC) : null, 'time' => '~' . round($time * 1000, 2) . 'ms' ); $debug['time']['db_queries'] += $time; @@ -118,6 +127,9 @@ function query($query) { sql_open(); if ($config['debug']) { + if ($config['debug_explain'] && preg_match('/^(SELECT|INSERT|UPDATE|DELETE) /', $query)) { + $explain = $pdo->query("EXPLAIN $query") or error(db_error()); + } $start = microtime(true); $query = $pdo->query($query); if (!$query) @@ -126,6 +138,7 @@ function query($query) { $debug['sql'][] = array( 'query' => $query->queryString, 'rows' => $query->rowCount(), + 'explain' => isset($explain) ? $explain->fetchAll(PDO::FETCH_ASSOC) : null, 'time' => '~' . round($time * 1000, 2) . 'ms' ); $debug['time']['db_queries'] += $time; -- cgit v1.3 From b7f16dee0f4554f5e607e50f773af6852e716513 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sat, 7 Sep 2013 12:58:23 +1000 Subject: Add ! syntax (NOT) to filters. Don't throttle duplicate post bodies when they are empty --- inc/config.php | 7 ++++--- inc/filters.php | 9 ++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) (limited to 'inc/config.php') diff --git a/inc/config.php b/inc/config.php index 43550dc3..9f231e30 100644 --- a/inc/config.php +++ b/inc/config.php @@ -305,7 +305,7 @@ $config['filters'][] = array( 'condition' => array( 'flood-match' => array('ip'), // Only match IP address - 'flood-time' => &$config['flood_time'] // 10 seconds minimum + 'flood-time' => &$config['flood_time'] ), 'action' => 'reject', 'message' => &$config['error']['flood'] @@ -315,7 +315,8 @@ $config['filters'][] = array( 'condition' => array( 'flood-match' => array('ip', 'body'), // Match IP address and post body - 'flood-time' => &$config['flood_time_ip'] // 2 minutes minimum + 'flood-time' => &$config['flood_time_ip'], + '!body' => '/^$/', // Post body is NOT empty ), 'action' => 'reject', 'message' => &$config['error']['flood'] @@ -325,7 +326,7 @@ $config['filters'][] = array( 'condition' => array( 'flood-match' => array('body'), // Match IP address and post body - 'flood-time' => &$config['flood_time_same'] // 30 seconds minimum + 'flood-time' => &$config['flood_time_same'] ), 'action' => 'reject', 'message' => &$config['error']['flood'] diff --git a/inc/filters.php b/inc/filters.php index 27b167ea..5c934532 100644 --- a/inc/filters.php +++ b/inc/filters.php @@ -168,11 +168,14 @@ class Filter { public function check(array $post) { foreach ($this->condition as $condition => $value) { - if (!$this->match($post, $condition, $value)) + if ($condition[0] == '!') { + $NOT = true; + $condition = substr($condition, 1); + } else $NOT = false; + + if ($this->match($post, $condition, $value) == $NOT) return false; } - - /* match */ return true; } } -- cgit v1.3 From a13571cdade07802fcf8c72cb0924cf455032a12 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sun, 8 Sep 2013 14:59:43 +1000 Subject: Comment mistake --- inc/config.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'inc/config.php') diff --git a/inc/config.php b/inc/config.php index 9f231e30..0cdbe709 100644 --- a/inc/config.php +++ b/inc/config.php @@ -325,7 +325,7 @@ // Minimum time between posts with the same text. (Same content, but not always the same IP address.) $config['filters'][] = array( 'condition' => array( - 'flood-match' => array('body'), // Match IP address and post body + 'flood-match' => array('body'), // Match only post body 'flood-time' => &$config['flood_time_same'] ), 'action' => 'reject', -- cgit v1.3 From d4cf4c7afb975cead41768c9e0d1960a46183409 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sun, 8 Sep 2013 15:07:55 +1000 Subject: flood-count condition --- inc/config.php | 19 ++++++++++++++++--- inc/filters.php | 8 ++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) (limited to 'inc/config.php') diff --git a/inc/config.php b/inc/config.php index 0cdbe709..0eac92b6 100644 --- a/inc/config.php +++ b/inc/config.php @@ -342,7 +342,20 @@ // 'message' => &$config['error']['flood'] // ); - // An example of blocking an imaginary known spammer, who keeps posting a reply with the name "surgeon", + // Example: Use the "flood-count" condition to only match if the user has made at least two posts with + // the same content and IP address in the past 2 minutes. + // $config['filters'][] = array( + // 'condition' => array( + // 'flood-match' => array('ip', 'body'), // Match IP address and post body + // 'flood-time' => 60 * 2, // 2 minutes + // 'flood-count' => 2 // At least two recent posts + // ), + // '!body' => '/^$/', + // 'action' => 'reject', + // 'message' => &$config['error']['flood'] + // ); + + // Example: Blocking an imaginary known spammer, who keeps posting a reply with the name "surgeon", // ending his posts with "regards, the surgeon" or similar. // $config['filters'][] = array( // 'condition' => array( @@ -354,7 +367,7 @@ // 'message' => 'Go away, spammer.' // ); - // Same as above, but issuing a 3-hour ban instead of just reject the post. + // Example: Same as above, but issuing a 3-hour ban instead of just reject the post. // $config['filters'][] = array( // 'condition' => array( // 'name' => '/^surgeon$/', @@ -366,7 +379,7 @@ // 'reason' => 'Go away, spammer.' // ); - // PHP 5.3+ (anonymous functions) + // Example: PHP 5.3+ (anonymous functions) // There is also a "custom" condition, making the possibilities of this feature pretty much endless. // This is a bad example, because there is already a "name" condition built-in. // $config['filters'][] = array( diff --git a/inc/filters.php b/inc/filters.php index d2802264..d5353afb 100644 --- a/inc/filters.php +++ b/inc/filters.php @@ -73,6 +73,14 @@ class Filter { } } return false; + case 'flood-count': + $count = 0; + foreach ($this->flood_check as $flood_post) { + if (time() - $flood_post['time'] <= $this->condition['flood-time']) { + ++$count; + } + } + return $count >= $match; case 'name': return preg_match($match, $post['name']); case 'trip': -- cgit v1.3 From 9a846d5ad5838d4db824623d9916d6b4e34c29ee Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Sun, 8 Sep 2013 17:01:55 +1000 Subject: Use Unicode in antispam stuff --- inc/anti-bot.php | 42 +++++++++++++++++++++++++++++------------- inc/config.php | 3 +++ 2 files changed, 32 insertions(+), 13 deletions(-) (limited to 'inc/config.php') diff --git a/inc/anti-bot.php b/inc/anti-bot.php index dbe87e2b..59f5351f 100644 --- a/inc/anti-bot.php +++ b/inc/anti-bot.php @@ -11,14 +11,19 @@ $hidden_inputs_twig = array(); class AntiBot { public $salt, $inputs = array(), $index = 0; - public static function randomString($length, $uppercase = false, $special_chars = false) { + public static function randomString($length, $uppercase = false, $special_chars = false, $unicode_chars = false) { $chars = 'abcdefghijklmnopqrstuvwxyz0123456789'; if ($uppercase) $chars .= 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'; if ($special_chars) $chars .= ' ~!@#$%^&*()_+,./;\'[]\\{}|:<>?=-` '; + if ($unicode_chars) { + $len = strlen($chars) / 10; + for ($n = 0; $n < $len; $n++) + $chars .= mb_convert_encoding('&#' . mt_rand(0x2600, 0x26FF) . ';', 'UTF-8', 'HTML-ENTITIES'); + } - $chars = str_split($chars); + $chars = preg_split('//u', $chars, -1, PREG_SPLIT_NO_EMPTY); $ch = array(); @@ -41,10 +46,10 @@ class AntiBot { } public static function make_confusing($string) { - $chars = str_split($string); + $chars = preg_split('//u', $string, -1, PREG_SPLIT_NO_EMPTY); foreach ($chars as &$c) { - if (rand(0, 2) != 0) + if (mt_rand(0, 3) != 0) $c = utf8tohtml($c); else $c = mb_encode_numericentity($c, array(0, 0xffff, 0, 0xffff), 'UTF-8'); @@ -65,13 +70,13 @@ class AntiBot { shuffle($config['spam']['hidden_input_names']); - $input_count = rand($config['spam']['hidden_inputs_min'], $config['spam']['hidden_inputs_max']); + $input_count = mt_rand($config['spam']['hidden_inputs_min'], $config['spam']['hidden_inputs_max']); $hidden_input_names_x = 0; for ($x = 0; $x < $input_count ; $x++) { - if ($hidden_input_names_x === false || rand(0, 2) == 0) { + if ($hidden_input_names_x === false || mt_rand(0, 2) == 0) { // Use an obscure name - $name = $this->randomString(rand(10, 40)); + $name = $this->randomString(mt_rand(10, 40), false, false, $config['spam']['unicode']); } else { // Use a pre-defined confusing name $name = $config['spam']['hidden_input_names'][$hidden_input_names_x++]; @@ -79,25 +84,33 @@ class AntiBot { $hidden_input_names_x = false; } - if (rand(0, 2) == 0) { + if (mt_rand(0, 2) == 0) { // Value must be null $this->inputs[$name] = ''; - } elseif (rand(0, 4) == 0) { + } elseif (mt_rand(0, 4) == 0) { // Numeric value - $this->inputs[$name] = (string)rand(0, 100); + $this->inputs[$name] = (string)mt_rand(0, 100000); } else { // Obscure value - $this->inputs[$name] = $this->randomString(rand(5, 100), true, true); + $this->inputs[$name] = $this->randomString(mt_rand(5, 100), true, true, $config['spam']['unicode']); } } } + public static function space() { + if (mt_rand(0, 3) != 0) + return ' '; + return str_repeat(' ', mt_rand(1, 3)); + } + public function html($count = false) { global $config; $elements = array( '', '', + '', + '', '', '', '', @@ -110,7 +123,7 @@ class AntiBot { $html = ''; if ($count === false) { - $count = rand(1, count($this->inputs) / 15); + $count = mt_rand(1, abs(count($this->inputs) / 15) + 1); } if ($count === true) { @@ -125,6 +138,9 @@ class AntiBot { $element = false; while (!$element) { $element = $elements[array_rand($elements)]; + $element = str_replace(' ', self::space(), $element); + if (mt_rand(0, 5) == 0) + $element = str_replace('>', self::space() . '>', $element); if (strpos($element, 'textarea') !== false && $value == '') { // There have been some issues with mobile web browsers and empty