<?php
namespace WPDRMS\ASP\Index;

use WPDRMS\Utils\Html;
use WPDRMS\ASP\Utils\Inflect\InflectController;
use WPDRMS\Utils\MB;
use WPDRMS\Utils\Str;

/**
 * @phpstan-type TokenizerArgs array{
 *      min_word_length: int,
 *      use_stopwords: bool,
 *      stopwords: string[],
 *      inflect: bool,
 *      language: string,
 *      extract_numeric: bool,
 *  }
 */
class Tokenizer {
	/**
	 * @var string unique random string for special replacements
	 */
	private string $randstr = 'wpivdny3htnydqd6mlyg';

	/**
	 * @var TokenizerArgs
	 */
	private array $args;

	/**
	 * @var string[]
	 */
	public static array $additional_keywords_pattern = array(
		'"',
		"'",
		'`',
		'’',
		'‘',
		'”',
		'“',
		'«',
		'»',
		'+',
		'.',
		',',
		':',
		'-',
		'_',
		'=',
		'%',
		'(',
		')',
		'{',
		'}',
		'*',
		'[',
		']',
		'|',
		'&',
		'/',
	);

	/**
	 * @param TokenizerArgs $args
	 */
	public function __construct( array $args ) {
		$defaults   = array(
			'min_word_length' => 2,
			'use_stopwords'   => false,
			'stopwords'       => array(),
			'inflect'         => false,
			'language'        => 'english',
			'extract_numeric' => true,
		);
		$this->args = wp_parse_args( $args, $defaults ); // @phpstan-ignore-line
	}

	public function setLanguage(string $language): void {
		$this->args['language'] = $language;
	}

	/**
	 * Performs a simple trimming, character removal on a string
	 *
	 * @param $str
	 * @param $post
	 * @return string
	 */
	function tokenizeSimple( $str, $post ): string {
		if ( function_exists( 'mb_internal_encoding' ) ) {
			mb_internal_encoding( 'UTF-8' );
		}

		$str = Str::anyToString($str);
		$str = Html::toTxt( $str );
		$str = strip_tags( $str );
		$str = stripslashes( $str );
		// Non breakable spaces to regular spaces
		$str = preg_replace('/\xc2\xa0/', ' ', $str);
		$str = preg_replace( '/[[:space:]]+/', ' ', $str );

		$str = str_replace( array( "\n", "\r", '  ' ), ' ', $str );
		// Turkish uppercase I does not lowercase correctly
		$str = str_replace( 'İ', 'i', $str );
		$str = MB::strtolower( $str );
		$str = trim($str);

		$str = Content::hebrewUnvocalize($str);
		$str = Content::arabicRemoveDiacritics($str);

		$negative_keywords = $this->getNegativeWords($post);
		foreach ( $negative_keywords as $negative_keyword ) {
			// If there is a negative keyword within, this case is over
			if ( strpos($str, $negative_keyword) !== false ) {
				return '';
			}
		}

		$stop_words = $this->getStopWords();
		foreach ( $stop_words as $stop_word ) {
			// whole word matches only
			if ( preg_match('/\b' . $stop_word . '\b/', $str, $match, PREG_OFFSET_CAPTURE) ) {
				return '';
			}
		}

		return $str;
	}

	/**
	 * Performs a simple trimming, character removal on a string, but returns array of keywords
	 * by the separator character
	 *
	 * @param $str
	 * @param $post
	 * @return array
	 */
	function tokenizePhrases( $str, $post, $word_separator = ',' ): array {
		if ( function_exists( 'mb_internal_encoding' ) ) {
			mb_internal_encoding( 'UTF-8' );
		}

		$args = $this->args;

		$str = Str::anyToString($str);
		$str = Html::toTxt( $str );
		$str = strip_tags( $str );
		$str = stripslashes( $str );
		// Non breakable spaces to regular spaces
		$str = preg_replace('/\xc2\xa0/', ' ', $str);
		$str = preg_replace( '/[[:space:]]+/', ' ', $str );

		$str = str_replace( array( "\n", "\r", '  ' ), ' ', $str );
		// Turkish uppercase I does not lowercase correctly
		$str = str_replace( 'İ', 'i', $str );
		$str = MB::strtolower( $str );
		$str = trim($str);

		$str = Content::hebrewUnvocalize($str);
		$str = Content::arabicRemoveDiacritics($str);

		$words = explode($word_separator, $str);
		$words = array_map('trim', $words);
		$words = array_filter(
			$words,
			function ( $word ) {
				return \WPDRMS\Utils\MB::strlen($word);
			}
		);

		$keywords = array();

		while ( ( $c_word = array_shift($words) ) !== null ) {
			$c_word = trim($c_word);

			if ( $c_word == '' || MB::strlen($c_word) < $args['min_word_length'] ) {
				continue;
			}

			// Numerics won't work otherwise, need to trim that later
			if ( is_numeric($c_word) ) {
				$c_word = ' ' . $c_word;
			}

			if ( array_key_exists($c_word, $keywords) ) {
				++$keywords[ $c_word ][1];
			} else {
				$keywords[ $c_word ] = array( $c_word, 1 );
			}
		}
		unset($c_word);

		return $keywords;
	}

	/**
	 * Performs a keyword extraction on the given content string.
	 *
	 * @return array of keywords $keyword = array( 'keyword', {count} )
	 */
	function tokenize( $str, $post = false, $lang = '' ): array {

		if ( is_array( $str ) ) {
			$str = Str::anyToString( $str );
		}
		if ( function_exists('mb_strlen') ) {
			$fn_strlen = 'mb_strlen';
		} else {
			$fn_strlen = 'strlen';
		}

		$args = $this->args;

		if ( function_exists( 'mb_internal_encoding' ) ) {
			mb_internal_encoding( 'UTF-8' );
		}

		$str = apply_filters( 'asp_indexing_string_pre_process', $str );

		$str = Html::toTxt( $str );
		$str = wp_specialchars_decode( $str );

		$str = strip_tags( $str );
		$str = stripslashes( $str );

		// Replace non-word boundary dots with a unique string + 'd'
		/** @noinspection RegExpRedundantEscape */
		$str = preg_replace('/([0-9])[\.]([0-9])/', '$1' . $this->randstr . 'd$2', $str);

		// Remove potentially dangerous or unusable characters
		$str = str_replace(
			array(
				'Â·',
				'â€¦',
				'â‚¬',
				'&shy;',
				'·',
				'…',
				'®',
				'©',
				'™',
				"\xC2\xAD",
			),
			'',
			$str 
		);
		$str = str_replace(
			array(
				'. ', // dot followed by space as boundary, otherwise it might be a part of the word
				', ', // comma followed by space only, otherwise it might be a word part
				'<',
				'>',
				'†',
				'‡',
				'‰',
				'‹',
				'™',
				'¡',
				'¢',
				'¤',
				'¥',
				'¦',
				'§',
				'¨',
				'©',
				'ª',
				'«',
				'¬',
				'®',
				'¯',
				'°',
				'±',
				'¹',
				'²',
				'³',
				'¶',
				'·',
				'º',
				'»',
				'¼',
				'½',
				'¾',
				'¿',
				'÷',
				'•',
				'…',
				'←',
				'←',
				'↑',
				'→',
				'↓',
				'↔',
				'↵',
				'⇐',
				'⇑',
				'⇒',
				'⇓',
				'⇔',
				'√',
				'∝',
				'∞',
				'∠',
				'∧',
				'∨',
				'∂',
				'∃',
				'∅',
				'∗',
				'∩',
				'∪',
				'∫',
				'∴',
				'∼',
				'≅',
				'≈',
				'≠',
				'≡',
				'≤',
				'≥',
				'⊂',
				'⊃',
				'⊄',
				'⊆',
				'⊇',
				'⊕',
				'⊗',
				'⊥',
				'◊',
				'♠',
				'♣',
				'♥',
				'♦',
				'🔴',
				'​',
				'◊',
				'〈',
				'〉',
				'⌊',
				'⌋',
				'⌈',
				'⌉',
				'⋅',
				'Ă‹â€ˇ',
				'Ă‚Â°',
				'~',
				'Ă‹â€ş',
				'Ă‹ĹĄ',
				'Ă‚Â¸',
				'Ă‚Â§',
				'Ă‚Â¨',
				'â€™',
				'â€',
				'â€ť',
				'â€ś',
				'â€ž',
				'Â´',
				'â€”',
				'â€“',
				'Ă—',
				'&#8217;',
				'&#128308;',
				'&nbsp;',
				"\n",
				"\r",
				'& ',
				'\\',
				'^',
				'?',
				'!',
				';',
				chr( 194 ) . chr( 160 ),
			),
			' ',
			$str 
		);
		$str = str_replace( 'Ăź', 'ss', $str );

		// Turkish uppercase I does not lowercase correctly
		$special_replace = array(
			'İ' => 'i',
			'—' => '-',
		);
		$str             = str_replace( array_keys($special_replace), array_values($special_replace), $str );

		// Any yet undefined punctuation
		// $str = preg_replace( '/[[:punct:]]+/u', ' ', $str );
		// Non breakable spaces to regular spaces
		$str = preg_replace('/\xc2\xa0/', ' ', $str);
		// Any remaining multiple space characters
		$str = preg_replace( '/[[:space:]]+/', ' ', $str );

		$str = MB::strtolower($str);

		$str = Content::hebrewUnvocalize($str);
		$str = Content::arabicRemoveDiacritics($str);

		// $str = preg_replace('/[^\p{L}0-9 ]/', ' ', $str);
		$str = str_replace( "\xEF\xBB\xBF", '', $str );

		$str = trim( preg_replace( '/\s+/', ' ', $str ) );

		// Set back the non-word boundary dots
		$str = str_replace( $this->randstr . 'd', '.', $str );

		$str = apply_filters( 'asp_indexing_string_post_process', $str );

		$words = explode( ' ', $str );

		// Remove punctuation marks + some extra from the start and the end of words

		// Characters, which should not be standalone (but can be in start on end)
		$non_standalone_strings = array( '$', '€', '£', '%' );
		// Additional keywords, should not be standalone
		$additional_keywords_string = implode('', array_diff(self::$additional_keywords_pattern, $non_standalone_strings));
		foreach ( $words as $wk => &$ww ) {
			$ww = MB::trim($ww, $additional_keywords_string);
			if ( $ww == '' || in_array($ww, $non_standalone_strings ) ) {
				unset($words[ $wk ]);
			}
		}
		unset($wk);
		unset($ww);

		// Get additional words if available
		$additional_words = array();
		$started          = microtime(true);
		foreach ( $words as $ww ) {
			// This operation can be costly, so limit to 3 seconds just to be sure
			if ( ( microtime(true) - $started ) > 3 ) {
				break;
			}

			// ex.: 123-45-678 to 123, 45, 678
			$ww1 = str_replace(self::$additional_keywords_pattern, ' ', $ww);
			$wa  = explode(' ', $ww1);
			if ( count($wa) > 1 ) {
				foreach ( $wa as $wak => $wav ) {
					$wav = trim(preg_replace( '/[[:space:]]+/', ' ', $wav ));
					if ( $wav != '' && !in_array($wav, $words) ) {
						$wa[ $wak ] = $wav;
					} else {
						unset($wa[ $wak ]);
					}
				}
				$additional_words = array_merge($additional_words, $wa);
			}
			// ex.: 123-45-678 to 12345678
			$ww2 = str_replace(self::$additional_keywords_pattern, '', $ww);
			if ( $ww2 !== '' && $ww2 != $ww && !in_array($ww2, $words) && !in_array($ww2, $additional_words) ) {
				$additional_words[] = $ww2;
			}

			// Accent removal and transliteration
			$transliterated = str_replace(array( 'ʾ', "'", '"' ), '', Str::removeAccents($ww));
			if (
				$transliterated !== '' &&
				$transliterated !== $ww &&
				!in_array($transliterated, $words, true) &&
				!in_array($transliterated, $additional_words, true)
			) {
				$additional_words[] = $transliterated;
			}
		}

		// Append them after the words array
		$words = array_merge($words, $additional_words);
		// Inflections
		if ( $this->args['inflect'] && !empty($words) ) {
			$words = array_merge(
				$words,
				InflectController::instance()->get($words, $this->args['language'])
			);
		}

		/**
		 * Apply synonyms for the whole string instead of the words, because
		 * synonyms can be multi-keyword phrases too
		 */
		$syn_inst = \WPDRMS\ASP\Synonyms\SynonymsService::getInstance();
		if ( $syn_inst->exists() ) {
			$syn_inst->synonymsAsKeywords();
			$additional_words_by_synonyms = array();
			$synonyms                     = $syn_inst->get();

			// If the langauge is set
			if ( $lang != '' && isset($synonyms[ $lang ]) ) {
				foreach ( $synonyms[ $lang ] as $keyword => $synonyms_arr ) {
					if ( preg_match('/\b' . preg_quote($keyword) . '\b/u', $str) ) {
						$additional_words_by_synonyms = array_merge($additional_words_by_synonyms, $synonyms_arr);
					}
				}
			}
			unset($keyword, $synonyms_arr);

			// Also for the "default" aka "any"
			if ( isset($synonyms['default']) ) {
				foreach ( $synonyms['default'] as $keyword => $synonyms_arr ) {
					if ( preg_match('/\b' . preg_quote($keyword) . '\b/u', $str) ) {
						$additional_words_by_synonyms = array_merge($additional_words_by_synonyms, $synonyms_arr);
					}
				}
			}

			if ( count($additional_words_by_synonyms) > 0 ) {
				$words = array_merge($words, $additional_words_by_synonyms);
			}
		}

		$stop_words        = $this->getStopWords();
		$negative_keywords = $this->getNegativeWords($post);

		$keywords = array();

		foreach ( $words as $c_word ) {
			$c_word = trim($c_word);

			if ( $c_word == '' || $fn_strlen($c_word) < $args['min_word_length'] ) {
				continue;
			}

			// Only whole word matches for stop-words
			if ( !empty($stop_words) && in_array($c_word, $stop_words) ) {
				continue;
			}

			// Partial matches for negative keywords
			foreach ( $negative_keywords as $negative_keyword ) {
				if ( strpos($c_word, $negative_keyword) !== false ) {
					continue 2;
				}
			}

			// Numerics won't work otherwise, need to trim that later
			if ( is_numeric($c_word) ) {
				$c_word = ' ' . $c_word;
			}

			if ( array_key_exists($c_word, $keywords) ) {
				++$keywords[ $c_word ][1];
			} else {
				$keywords[ $c_word ] = array( $c_word, 1 );
			}
		}
		unset($c_word);

		if ( $this->args['extract_numeric'] ) {
			$keywords = self::extractNumericKeywords( $keywords, (int) $args['min_word_length'] );
		}

		return apply_filters( 'asp_indexing_keywords', $keywords );
	}

	/**
	 * Extracts the numeric runs from each keyword and adds them as separate keywords, so a prefix
	 * search like "123" can match a token such as "SKU1234" (via the extracted "1234"). Pure-numeric
	 * keywords are stored with a leading space to avoid PHP numeric array keys (trimmed before the DB
	 * write); this mirrors that convention. Runs shorter than $min_word_length, and runs equal to the
	 * whole keyword, are skipped.
	 *
	 * @param array<string, array{0:string,1:int}> $keywords
	 * @param int                                  $min_word_length
	 * @return array<string, array{0:string,1:int}>
	 */
	public static function extractNumericKeywords( array $keywords, int $min_word_length = 1 ): array {
		foreach ( $keywords as $data ) {
			$source = trim( $data[0] );
			if ( !preg_match_all( '!\d+\.*\d*!', $source, $matches ) ) {
				continue;
			}
			foreach ( $matches[0] as $number ) {
				$number = rtrim( $number, '.' );
				if ( $number === '' || $number === $source || strlen( $number ) < $min_word_length ) {
					continue;
				}
				$key = ' ' . $number;
				if ( array_key_exists( $key, $keywords ) ) {
					++$keywords[ $key ][1];
				} else {
					$keywords[ $key ] = array( $key, 1 );
				}
			}
		}
		return $keywords;
	}


	/**
	 * Returns the stop words
	 */
	private function getStopWords(): array {
		$stop_words = array();
		// Only compare to common words if $restrict is set to false
		if ( $this->args['use_stopwords'] == 1 && $this->args['stopwords'] != '' ) {
			$this->args['stopwords'] = str_replace(' ', '', $this->args['stopwords']);
			$stop_words              = explode( ',', $this->args['stopwords'] );
		}
		$stop_words = array_unique( $stop_words );
		foreach ( $stop_words as $sk => &$sv ) {
			$sv = trim($sv);
			if ( $sv == '' || MB::strlen($sv) < $this->args['min_word_length'] ) {
				unset($stop_words[ $sk ]);
			}
		}

		return $stop_words;
	}

	/**
	 * Returns negative keywords for the current post object
	 */
	private function getNegativeWords( $post ): array {
		// Post level stop-words, negative keywords
		if ( $post !== false ) {
			$negative_keywords = get_post_meta($post->ID, '_asp_negative_keywords', true);
			if ( !empty($negative_keywords) ) {
				$negative_keywords = trim( preg_replace('/\s+/', ' ', $negative_keywords) );
				$negative_keywords = explode(' ', $negative_keywords);
				$negative_keywords = array_filter($negative_keywords, fn( $keyword )=>$keyword !=='');
				return array_unique($negative_keywords);
			}
		}

		return array();
	}
}
