<?php

namespace WPDRMS\ASP\Index;

use WPDRMS\ASP\Index\ORM\IndexSkipTable;
use WPDRMS\ASP\Index\ORM\IndexTable;
use WPDRMS\ASP\Media\Parser;
use WPDRMS\ASP\Media\RemoteService\License;
use WPDRMS\Utils\Html;
use WPDRMS\Utils\MB;
use WPDRMS\Utils\Str;

defined( 'ABSPATH' ) or die( "You can't access this file directly." );


class IndexProcessor {

	/** @var array<string, mixed> */
	protected array $args;

	private Shortcode $shortcode;

	private int $keywords_found    = 0;
	private int $posts_indexed_now = 0;
	private float $start_time      = 0.0;
	private bool $abort            = false;
	private string $lang           = '';

	/** @var Tokenizer */
	private Tokenizer $tokenizer;

	/** @var string[] */
	public static array $apostrophes = array( '"', "'", '`', '’', '‘', '”', '“' );

	// ------------------------------------------- PUBLIC METHODS --------------------------------------------------

	public function __construct( array $args = array() ) {

		$defaults = array(
			// Arguments here
			'index_title'                    => 1,
			'index_content'                  => 1,
			'index_pdf_content'              => 0,
			'index_pdf_method'               => 'auto',
			'index_text_content'             => 0,
			'index_richtext_content'         => 0,
			'index_msword_content'           => 0,
			'index_msexcel_content'          => 0,
			'index_msppt_content'            => 0,
			'media_service_send_file'        => 1,
			'index_excerpt'                  => 1,
			'index_tags'                     => 0,
			'index_categories'               => 0,
			'index_taxonomies'               => '',
			'attachment_mime_types'          => '',
			'attachment_exclude_directories' => array(),
			'attachment_include_directories' => array(),
			'index_permalinks'               => 0,
			'index_customfields'             => '',
			'index_author_name'              => '',
			'index_author_bio'               => '',
			'blog_id'                        => get_current_blog_id(),
			'inflections'                    => 0,
			'language'                       => 'english',
			'extend'                         => 1,
			'limit'                          => 25,
			'use_stopwords'                  => 1,
			'stopwords'                      => '',
			'min_word_length'                => 3,
			'post_types'                     => array( 'post', 'page' ),
			'post_statuses'                  => 'publish',
			'extract_gutenberg_blocks'       => 1,
			'extract_shortcodes'             => 1,
			'exclude_shortcodes'             => '',
			'extract_iframes'                => 0,
			'enable_compression'             => false,
			'cf_exclusion_rules'             => array(),
			'cf_exclusion_rules_logic'       => 'AND',
			'taxonomy_exclusion_rules'       => array(),
			'taxonomy_exclusion_rules_logic' => 'AND',
		);

		$this->args                                   = wp_parse_args( $args, $defaults );
		$this->args['attachment_exclude_directories'] = array_filter($this->args['attachment_exclude_directories']);
		$this->args['attachment_include_directories'] = array_filter($this->args['attachment_include_directories']);
		$this->args                                   = apply_filters( 'asp_it_args', $this->args, $defaults);

		$this->shortcode = new Shortcode();
		$this->tokenizer = new Tokenizer(
			array(
				'min_word_length' => $this->args['min_word_length'],
				'use_stopwords'   => $this->args['use_stopwords'],
				'stopwords'       => $this->args['stopwords'],
				'inflect'         => $this->args['inflections'],
				'language'        => $this->args['language'],
				'extract_numeric' => $this->args['extract_numeric'] ?? true,
			)
		);

		// Swap here to have the asp_posts_indexed option for each blog different
		if ( is_multisite() && !empty($this->args['blog_id']) && $this->args['blog_id'] != get_current_blog_id() ) {
			switch_to_blog( $this->args['blog_id'] );
		}

		do_action('asp_index_init');
	}

	/**
	 * Re-generates the index table
	 *
	 * @return array (posts to index, posts indexed)
	 */
	public function newIndex(): array {
		$this->beforeIndex();
		return $this->extendIndex();
	}

	public function beforeIndex(): void {
		$this->emptyIndex( false );
		$this->emptyIgnoreList();
		delete_option( 'asp_index_cursor' );
		IndexTable::beforeIndex( $this->args['enable_compression'] );
	}


	/**
	 * Extends the index database
	 *
	 * @return array (posts to index, posts indexed)
	 */
	public function extendIndex(): array {

		do_action('asp_index_start');
		$this->start_time = microtime(true);
		usleep( IndexTableMitigation::getRestTime() );

		$cursor = $this->args['extend'] ? 0 : (int) get_option( 'asp_index_cursor', 0 );
		$posts  = IndexTable::getPostIdsToIndex( $this->args, $cursor );

		if ( ! $this->args['extend'] ) {
			if ( empty( $posts ) ) {
				delete_option( 'asp_index_cursor' );
				// No pending posts remain: a full (re)index has finished. Fires once per rebuild,
				// after the index is fully populated (not at the start), so consumers invalidating
				// index-derived caches never re-cache from a half-built table.
				do_action( 'asp_index_rebuild_complete' );
			} else {
				update_option(
					'asp_index_cursor',
					max(
						array_map(
							function ( $p ) {
								return (int) $p->ID;
							},
							$posts 
						) 
					) 
				);
			}
		}

		foreach ( $posts as $tpost ) {
			if ( $this->abort ) {
				break;
			}

			if ( $this->indexDocument( $tpost->ID, false ) ) {
				++$this->posts_indexed_now;
			}
		}

		// THIS MUST BE HERE!!
		// .the statment below restores the blog before getting the correct count!
		$return = array(
			'postsToIndex'    => $this->getPostIdsToIndexCount(),
			'postsIndexedNow' => $this->getPostsIndexedNow(),
			'keywordsFound'   => $this->keywords_found,
		);

		if ( is_multisite() ) {
			restore_current_blog();
		}

		return $return;
	}

	public function emptyIgnoreList(): void {
		delete_option( '_asp_index_ignore' );
	}

	/**
	 * Indexes a document based on its ID
	 *
	 * @param int  $post_id the post id
	 * @param bool $remove_first
	 * @param bool $post_editor_context
	 * @return bool
	 */
	public function indexDocument( int $post_id, bool $remove_first = true, bool $post_editor_context = false ): bool {
		$args = $this->args;

		// array of all needed tokens
		$tokens = array();

		// forbidden post types
		$forbidden_pt = array( 'tablepress_table', 'vc_grid_item', 'revision', 'nav_menu_item', 'custom_css', 'acf' );

		// On creating or extending the index, no need to remove
		if ( $remove_first ) {
			$this->removeDocument( $post_id );
		}

		/**
		 * This prevents the fancy quotes and special characters to HTML output
		 * NOTE: it has to be executed here before every get_post() call!!
		 */
		remove_filter('the_title', 'wptexturize');
		remove_filter('the_title', 'convert_chars');

		$the_post = get_post( $post_id );
		// Return false to skip the post from the index
		$the_post = apply_filters('asp_index_post', $the_post, $args);

		if ( $the_post == null ) {
			return false;
		}

		if ( apply_filters( 'asp/index/document/exclude', false, $the_post->ID, $the_post, $args ) ) {
			IndexSkipTable::markAsExcluded( $the_post->ID, (int) $args['blog_id'] );
			return false;
		}

		// This needs to be here, after the get_post()
		if ( $post_editor_context === true ) {
			if ( count($args['post_types']) ) {
				if ( !in_array($the_post->post_type, $args['post_types']) ) {
					return false;
				}
			} else {
				return false;
			}
		}

		// Is this a forbidden post type?
		if ( in_array($the_post->post_type, $forbidden_pt) ) {
			return false;
		}

		// Check if attachment, if so, check the mime types allowed
		if ( $the_post->post_type == 'attachment' ) {
			$mimes_arr = wpd_comma_separated_to_array($args['attachment_mime_types']);
			if ( !in_array($the_post->post_mime_type, $mimes_arr) ) {
				return false;
			}
		}

		// Password protected excluded
		if ( $args['post_password_protected'] == 0 && post_password_required($the_post) ) {
			return false;
		}

		if ( $this->shouldExcludeByTaxonomyRules( $the_post->ID ) ) {
			IndexSkipTable::markAsExcluded( $the_post->ID, (int) $args['blog_id'] );
			return false;
		}

		// Date / author / manual-post-ID exclusions (also applied as a SQL filter on the
		// bulk source query; this covers the incremental save_post path).
		if ( IndexExclusionEvaluator::excludesPost( $the_post, $args ) ) {
			IndexSkipTable::markAsExcluded( $the_post->ID, (int) $args['blog_id'] );
			return false;
		}

		// --- GET THE LANGUAGE INFORMATION, IF ANY ---
		$lang = '';
		// Is WPML used?
		if ( class_exists('SitePress') ) {
			$lang = $this->wpml_langcode_post_id( $the_post );
		}
		// Is Polylang used?
		if ( function_exists('pll_get_post_language') && $lang == '' ) {
			if ( $the_post->post_type == 'product_variation' && class_exists('WooCommerce') ) {
				/** @noinspection PhpRedundantOptionalArgumentInspection */
				$lang = pll_get_post_language($the_post->post_parent, 'slug');
			} else {
				/** @noinspection PhpRedundantOptionalArgumentInspection */
				$lang = pll_get_post_language($the_post->ID, 'slug');
			}
		}
		$this->lang = $lang;

		// Set the tokenizer to the content language
		if ( $this->args['inflections'] ) {
			switch ( $this->lang ) {
				case 'en':
					$this->tokenizer->setLanguage('english');
					break;
				case 'fr':
					$this->tokenizer->setLanguage('french');
					break;
				case 'no':
					$this->tokenizer->setLanguage('norwegian-bokmal');
					break;
				case 'pt':
					$this->tokenizer->setLanguage('portuguese');
					break;
				case 'es':
					$this->tokenizer->setLanguage('spanish');
					break;
				case 'tr':
					$this->tokenizer->setLanguage('turkish');
					break;
				default:
					$this->tokenizer->setLanguage($this->args['language']);
					break;
			}
		}

		/**
		 * For product variations set the title, content and excerpt to the original product
		 */
		if ( $the_post->post_type == 'product_variation' ) {
			$parent_post = get_post($the_post->post_parent);
			if ( !empty($parent_post) ) {
				$the_post->post_title  .= ' ' . $parent_post->post_title;
				$the_post->post_content = $parent_post->post_content;
				$the_post->post_excerpt = $parent_post->post_excerpt;
			}
		}

		if ( $args['index_content'] == 1 ) {
			$this->tokenizeContent( $the_post, $tokens );
		}

		if ( $args['index_title'] == 1 ) {
			$this->tokenizeTitle( $the_post, $tokens );
		}

		if ( $the_post->post_type == 'attachment' ) {
			$this->tokenizeMedia( $the_post, $tokens );
		}

		if ( $args['index_excerpt'] == 1 ) {
			$this->tokenizeExcerpt( $the_post, $tokens );
		}

		if ( $args['index_categories'] == 1 || $args['index_tags'] == 1 || $args['index_taxonomies'] != '' ) {
			$this->tokenizeTerms( $the_post, $tokens );
		}

		if ( $args['index_author_name'] == 1 || $args['index_author_bio'] == 1 ) {
			$this->tokenizeAuthor( $the_post, $tokens );
		}

		if ( $args['index_permalinks'] == 1 ) {
			$this->tokenizePermalinks( $the_post, $tokens );
		}

		if ( ! empty( $this->args['cf_exclusion_rules'] ) || $args['index_customfields'] !== '' ) {
			$all_meta = get_post_meta( $the_post->ID );
			if ( $this->shouldExcludeByCustomFieldRules( $all_meta ) ) {
				IndexSkipTable::markAsExcluded( $the_post->ID, (int) $args['blog_id'] );
				return false;
			}
			$this->tokenizeCustomFields( $the_post, $tokens, $all_meta );
		} else {
			$this->tokenizeCustomFields( $the_post, $tokens );
		}

		$this->tokenizeAdditionalKeywords( $the_post, $tokens );

		if ( count( $tokens ) > 0 ) {
			$this->keywords_found += IndexTable::insertTokensToDB( $the_post, $tokens, $args['blog_id'], $this->lang );
			return true;
		}

		/*
		DO NOT call finishOperation() here, it would switch back the blog too early.
		Calling this function from an action hooks does not require switching the blog,
		as the correct one is in use there.
		*/

		return false;
	}

	/**
	 * Removes a document from the index (in case of deleting posts, etc.)
	 *
	 * @param int|array $post_id the post id
	 */
	public function removeDocument( $post_id ): void {
		IndexTable::removeDocument( $post_id );
		IndexSkipTable::removeDocument( $post_id, (int) $this->args['blog_id'] );

		/*
		DO NOT call finishOperation() here, it would switch back the blog too early.
		Calling this function from an action hooks does not require switching the blog,
		as the correct one is in use there.
		*/
	}


	/**
	 * Empties the index table
	 *
	 * @param bool $restore_current_blog if set to false, it won't restore multiste blog - for internal usage mainly
	 * @return array
	 */
	public function emptyIndex( bool $restore_current_blog = true ): array {
		global $wpdb;
		IndexTable::truncateTable();

		if ( is_multisite() ) {
			$current = get_current_blog_id();
			$blogs   = $wpdb->get_results( "SELECT blog_id FROM $wpdb->blogs", ARRAY_A );
			if ( $blogs ) {
				foreach ( $blogs as $blog ) {
					switch_to_blog( $blog['blog_id'] );
				}
				// Switch back to the current, like nothing happened
				switch_to_blog( $current );
			}
		}

		if ( $restore_current_blog && is_multisite() ) {
			restore_current_blog();
		}

		return array(
			'postsToIndex'  => $this->getPostIdsToIndexCount(),
			'totalKeywords' => $this->getTotalKeywords(),
		);
	}

	/**
	 * Suggests pool sizes for the index table search process
	 *
	 * @return array
	 * @noinspection PhpUnused
	 */
	public static function suggestPoolSizes(): array {
		return array(
			'one'   => 50000,
			'two'   => 90000,
			'three' => 90000,
			'rest'  => 90000,
		);
	}


	// ------------------------------------------- PRIVATE METHODS -------------------------------------------------

	/**
	 * Checks whether a post should be excluded from the index based on CF exclusion rules.
	 *
	 * Rules are connected by the configured logic (AND = all must match, OR = at least one must match).
	 *
	 * @param array<string, array<int, string>> $all_meta Result of get_post_meta($id) — keys are meta keys, values are arrays of meta values.
	 * @return bool True if the post should be excluded, false otherwise.
	 */
	protected function shouldExcludeByCustomFieldRules( array $all_meta ): bool {
		$rules = isset( $this->args['cf_exclusion_rules'] ) ? $this->args['cf_exclusion_rules'] : array();
		if ( empty( $rules ) ) {
			return false;
		}
		$logic  = isset( $this->args['cf_exclusion_rules_logic'] ) ? $this->args['cf_exclusion_rules_logic'] : 'AND';
		$is_and = ( $logic !== 'OR' );

		foreach ( $rules as $rule ) {
			$field_key = isset( $rule['field_key'] ) ? $rule['field_key'] : '';
			$operator  = isset( $rule['operator'] ) ? $rule['operator'] : 'equals';
			$expected  = (string) ( isset( $rule['value'] ) ? $rule['value'] : '' );
			$actual    = isset( $all_meta[ $field_key ][0] ) ? (string) $all_meta[ $field_key ][0] : '';

			$match = false;
			switch ( $operator ) {
				case 'equals':
					$match = ( $actual === $expected );
					break;
				case 'not_equals':
					$match = ( $actual !== $expected );
					break;
				case 'contains':
					$match = ( strpos( $actual, $expected ) !== false );
					break;
				case 'not_contains':
					$match = ( strpos( $actual, $expected ) === false );
					break;
				case 'starts_with':
					$match = ( strpos( $actual, $expected ) === 0 );
					break;
				case 'ends_with':
					$match = ( $expected !== '' && substr( $actual, -strlen( $expected ) ) === $expected );
					break;
				case 'greater_than':
					$match = ( (float) $actual > (float) $expected );
					break;
				case 'less_than':
					$match = ( (float) $actual < (float) $expected );
					break;
				case 'is_empty':
					$match = ( $actual === '' );
					break;
				case 'is_not_empty':
					$match = ( $actual !== '' );
					break;
			}

			if ( $is_and && ! $match ) {
				return false; // AND: first mismatch → don't exclude
			}
			if ( ! $is_and && $match ) {
				return true;  // OR: first match → exclude immediately
			}
		}
		// AND: all matched → exclude. OR: no rule matched → don't exclude.
		return $is_and;
	}

	protected function shouldExcludeByTaxonomyRules( int $post_id ): bool {
		$rules = isset( $this->args['taxonomy_exclusion_rules'] ) ? $this->args['taxonomy_exclusion_rules'] : array();
		if ( empty( $rules ) ) {
			return false;
		}
		$logic     = isset( $this->args['taxonomy_exclusion_rules_logic'] ) ? $this->args['taxonomy_exclusion_rules_logic'] : 'AND';
		$is_and    = ( $logic !== 'OR' );
		$evaluated = false;

		foreach ( $rules as $rule ) {
			$taxonomy = isset( $rule['taxonomy'] ) ? $rule['taxonomy'] : '';
			$terms    = isset( $rule['terms'] ) ? (array) $rule['terms'] : array();
			if ( $taxonomy === '' || empty( $terms ) ) {
				continue;
			}
			$evaluated = true;
			$match     = has_term( $terms, $taxonomy, $post_id );

			if ( $is_and && ! $match ) {
				return false;
			}
			if ( ! $is_and && $match ) {
				return true;
			}
		}
		// If no valid rule was evaluated, never exclude.
		return $evaluated && $is_and;
	}

	/**
	 * Generates the content tokens and puts them into the tokens array
	 *
	 * @param object $the_post the post object
	 * @param array  $tokens tokens array
	 *
	 * @return int keywords count
	 * @noinspection PhpReturnValueOfMethodIsNeverUsedInspection
	 */
	private function tokenizeContent( $the_post, &$tokens ): int {
		$args = $this->args;

		// Some custom editors like Themeco X Pro requires the global $post object for it to work.
		global $post;
		$post = $the_post;

		$content = apply_filters( 'asp_post_content_before_tokenize_clear', $the_post->post_content, $the_post );

		if ( $args['extract_shortcodes'] ) {
			$content = $this->shortcode->execute( $content, $the_post, $args['exclude_shortcodes'] );
		}

		if ( $args['extract_gutenberg_blocks'] ) {
			$content = do_blocks( $content );
		}

		if ( $args['extract_iframes'] == 1 ) {
			$content .= ' ' . Html::extractIframeContent($content);
		}

		// Strip the remaining shortcodes
		$content = strip_shortcodes( $content );

		$filtered_content = apply_filters( 'asp_post_content_before_tokenize', $content, $the_post );

		if ( $filtered_content == '' ) {
			return 0;
		}

		$content_keywords = $this->tokenizer->tokenize( $filtered_content, $the_post, $this->lang );

		foreach ( $content_keywords as $keyword ) {
			/** @noinspection PhpRedundantOptionalArgumentInspection */
			$this->insertToken( $tokens, $keyword[0], $keyword[1], 'content' );
		}

		return count( $content_keywords );
	}

	/**
	 * Generates the excerpt tokens and puts them into the tokens array
	 *
	 * @param object $the_post the post object
	 * @param array  $tokens tokens array
	 *
	 * @return int keywords count
	 * @noinspection PhpReturnValueOfMethodIsNeverUsedInspection
	 */
	private function tokenizeExcerpt( $the_post, &$tokens ): int {
		$args = $this->args;

		if ( $the_post->post_excerpt == '' ) {
			return 0;
		}

		$filtered_excerpt = apply_filters( 'asp_post_excerpt_before_tokenize', $the_post->post_excerpt, $the_post );

		if ( $args['extract_shortcodes'] ) {
			$filtered_excerpt = $this->shortcode->execute( $filtered_excerpt, $the_post, $args['exclude_shortcodes'] );
		}

		$excerpt_keywords = $this->tokenizer->tokenize( $filtered_excerpt, $the_post, $this->lang );

		foreach ( $excerpt_keywords as $keyword ) {
			$this->insertToken( $tokens, $keyword[0], $keyword[1], 'excerpt' );
		}

		return count( $excerpt_keywords );
	}

	/**
	 * Generates the title tokens and puts them into the tokens array
	 *
	 * @param object $the_post the post object
	 * @param array  $tokens tokens array
	 *
	 * @return int keywords count
	 */
	private function tokenizeTitle( $the_post, &$tokens ): int {
		$filtered_title = apply_filters( 'asp_post_title_before_tokenize', $the_post->post_title, $the_post );
		// the_title filter causes issues with empty strings, so check
		if ( trim($filtered_title) == '' ) {
			return 0;
		}

		$title          = apply_filters( 'the_title', $filtered_title, $the_post->ID );
		$title_keywords = $this->tokenizer->tokenize( $title, $the_post, $this->lang );

		// No-reverse exact title
		$single_title = $this->tokenizer->tokenizeSimple($title, $the_post);
		if ( $single_title != '' ) {
			$single_title = MB::substr($single_title, 0, 45);
			$pos          = MB::strpos($single_title, ' ');

			/*
			 * The index table unique key is (doc, term, item) - so if this word already exists, then it will be ignored
			 * by the database. To make sure it is added, append a unique string at the end, that is not searched.
			 */
			if ( $pos === false ) {
				$single_title .= '___';
			}

			$this->insertToken($tokens, $single_title, 1, 'title', true);

			$single_title_al = str_replace(self::$apostrophes, '', $single_title);
			if ( $single_title_al !== $single_title ) {
				$this->insertToken($tokens, $single_title_al, 1, 'title', true);
			}

			$single_title_spaceless = str_replace(' ', '', $single_title_al);
			if ( $single_title_spaceless !== $single_title ) {
				$this->insertToken($tokens, $single_title_spaceless, 1, 'title', true);
			}
		}

		foreach ( $title_keywords as $keyword ) {
			$this->insertToken( $tokens, $keyword[0], $keyword[1], 'title' );
		}

		$post_type_obj = get_post_type_object($the_post->post_type);
		if ( !is_wp_error($post_type_obj) ) {
			if ( isset($post_type_obj->labels->name) ) {
				$post_type_name = $this->tokenizer->tokenizeSimple($post_type_obj->labels->name, $the_post);
				if ( $post_type_name != '' ) {
					$this->insertToken($tokens, $post_type_name, 1, 'title');
				}
			}
			if ( isset($post_type_obj->labels->singular_name) ) {
				$post_type_name = $this->tokenizer->tokenizeSimple($post_type_obj->labels->singular_name, $the_post);
				if ( $post_type_name != '' ) {
					$this->insertToken($tokens, $post_type_name, 1, 'title');
				}
			}
		}

		return count( $title_keywords );
	}

	/**
	 * Generates the media file contents depending on the media type
	 *
	 * @param $the_post - the post object
	 * @param $tokens - tokens array
	 *
	 * @return int - keywords count
	 * @noinspection PhpIncludeInspection
	 */
	private function tokenizeMedia( $the_post, &$tokens ): int {
		$args = $this->args;

		$p = new Parser(
			$the_post,
			array(
				'pdf_parser'              => $args['index_pdf_method'],
				'text_content'            => $args['index_text_content'],
				'richtext_content'        => $args['index_richtext_content'],
				'pdf_content'             => $args['index_pdf_content'],
				'epub_content'            => $args['index_epub_content'],
				'msword_content'          => $args['index_msword_content'],
				'msexcel_content'         => $args['index_msexcel_content'],
				'msppt_content'           => $args['index_msppt_content'],
				'media_service_send_file' => $args['media_service_send_file'],
			)
		);

		$this->abort = true; // Preemptively set the abort flag, so no other document content is indexed
		$contents    = apply_filters( 'asp_index_file_custom_parse', '', $the_post );

		if ( $contents == '' ) {

			$contents = $p->parse();

			if ( is_wp_error($contents) ) {
				if ( $contents->get_error_code() == 10 ) {
					License::getInstance()->deactivate();
				}
				$contents = '';
			}

			$time_elapsed_from_start = microtime(true) - $this->start_time;
			// Only abort if the whole indexing process took over 5 seconds so far
			if ( $time_elapsed_from_start < 5 ) {
				$this->abort = false;
			}
		}

		$contents = apply_filters( 'asp_file_contents_before_tokenize', $contents, $the_post );
		$keywords = $this->tokenizer->tokenize( $contents, $the_post, $this->lang );

		foreach ( $keywords as $keyword ) {
			/** @noinspection PhpRedundantOptionalArgumentInspection */
			$this->insertToken( $tokens, $keyword[0], $keyword[1], 'content' );
		}

		return count( $keywords );
	}

	/**
	 * Generates the permalink tokens and puts them into the tokens array
	 *
	 * @param object $the_post the post object
	 * @param array  $tokens tokens array
	 *
	 * @return int keywords count
	 */
	private function tokenizePermalinks( $the_post, &$tokens ): int {
		$filtered_permalink = apply_filters( 'asp_post_permalink_before_tokenize', $the_post->post_name, $the_post );
		// Store the permalink as is, with an occurence of 1
		$this->insertToken( $tokens, $filtered_permalink, 1, 'link' );

		return 1;
	}

	/**
	 * Generates the author display name and biography tokens and puts them into the tokens array
	 *
	 * @param object $the_post the post object
	 * @param array  $tokens tokens array
	 *
	 * @return int keywords count
	 */
	private function tokenizeAuthor( $the_post, &$tokens ): int {
		$args = $this->args;
		$bio  = '';

		$user = get_userdata($the_post->post_author);
		$name = $user->display_name . ' ' . $user->first_name . ' ' . $user->last_name . ' ' . $user->nickname;

		if ( $args['index_author_bio'] ) {
			$bio = get_user_meta( $the_post->post_author, 'description', true );
		}

		$author_keywords = $this->tokenizer->tokenize( $name . ' ' . $bio, $the_post, $this->lang );
		foreach ( $author_keywords as $keyword ) {
			$this->insertToken( $tokens, $keyword[0], $keyword[1], 'author' );
		}

		return count( $author_keywords );
	}

	/**
	 * Generates taxonomy term tokens and puts them into the tokens array
	 *
	 * @param object $the_post the post object
	 * @param array  $tokens tokens array
	 *
	 * @return int keywords count
	 */
	private function tokenizeTerms( $the_post, &$tokens ): int {
		$args       = $this->args;
		$taxonomies = array();
		$all_terms  = array();

		if ( $args['index_tags'] ) {
			$taxonomies[] = 'post_tag';
		}
		if ( $args['index_categories'] ) {
			$taxonomies[] = 'category';
		}
		$custom_taxonomies = explode( '|', $args['index_taxonomies'] );

		$taxonomies = array_merge( $taxonomies, $custom_taxonomies );

		foreach ( $taxonomies as $taxonomy ) {
			$terms = wp_get_post_terms( $the_post->ID, trim( $taxonomy ), array( 'fields' => 'names' ) );
			$terms = apply_filters('asp_index_terms', $terms, $taxonomy, $the_post);
			if ( is_array( $terms ) ) {
				$all_terms = array_merge( $all_terms, $terms );
			}
		}

		if ( count( $all_terms ) > 0 ) {
			foreach ( $all_terms as $term ) {
				// Multi-word taxonomy terms also being added as whole
				if ( strpos($term, ' ') !== false ) {
					$single_term = $this->tokenizer->tokenizeSimple($term, $the_post);
					if ( $single_term != '' ) {
						$this->insertToken( $tokens, $single_term, 1, 'tag' );
					}
				}
			}

			$terms_string  = implode( ' ', $all_terms );
			$term_keywords = $this->tokenizer->tokenize( $terms_string, $the_post, $this->lang );

			// everything goes under the tags, thus the tokinezer is called only once
			foreach ( $term_keywords as $keyword ) {
				$this->insertToken( $tokens, $keyword[0], $keyword[1], 'tag' );
			}

			return count( $term_keywords );
		}

		return 0;
	}

	private function tokenizeAdditionalKeywords( $the_post, &$tokens ): void {
		$values = get_post_meta( $the_post->ID, '_asp_additional_tags', false );
		$values = is_array($values) ? $values : array( $values );
		$values = apply_filters( 'asp_post_additional_keywords_before_tokenize', $values, $the_post, $tokens );
		if ( count($values) > 0 ) {
			$keywords = $this->tokenizer->tokenizePhrases($values, $the_post);
			foreach ( $keywords as $keyword ) {
				$this->insertToken($tokens, $keyword[0], $keyword[1], 'customfield');
			}

			$keywords = $this->tokenizer->tokenize($values, $the_post, $this->lang);
			foreach ( $keywords as $keyword ) {
				$this->insertToken($tokens, $keyword[0], $keyword[1], 'customfield');
			}
		}
	}

	/**
	 * Generates selected custom field tokens and puts them into the tokens array
	 *
	 * @param object                                 $the_post the post object
	 * @param array                                  $tokens tokens array
	 * @param array<string, array<int, string>>|null $all_meta Optional pre-fetched result of get_post_meta($id). When provided, avoids redundant DB calls per field.
	 *
	 * @return int keywords count
	 */
	protected function tokenizeCustomFields( $the_post, &$tokens, ?array $all_meta = null ): int {
		$args = $this->args;

		if ( function_exists('mb_strlen') ) {
			$fn_strlen = 'mb_strlen';
		} else {
			$fn_strlen = 'strlen';
		}

		// all the CF content to this variable
		$cf_content = '';

		if ( $args['index_customfields'] != '' ) {
			$custom_fields = explode( '|', $args['index_customfields'] );
		} else {
			$custom_fields = array();
		}

		if ( in_array('_asp_additional_tags', $custom_fields) ) {
			$custom_fields = array_diff($custom_fields, array( '_asp_additional_tags' ));
		}

		foreach ( $custom_fields as $field ) {
			$values = array();
			if ( strpos($field, '__pods__') !== false ) {
				$field = str_replace('__pods__', '', $field);
				if ( function_exists('pods') ) {
					$p = pods($the_post->post_type, $the_post->ID);
					if ( is_object($p) ) {
						$values = $p->field($field, false);
					}
				}
			}

			$field_type = function_exists('get_field') ? asp_acf_get_field_type($field) : 'text';

			if ( empty($values) ) {
				if ( $all_meta !== null && isset( $all_meta[ $field ] ) ) {
					$values = $all_meta[ $field ];
				} else {
					/**
					 * Keys missing from the pre-fetched postmeta may still be served through the
					 * get_post_metadata filter (Jet Engine external storage etc.) — only a per-key
					 * call goes through that filter, the full-table pre-fetch does not.
					 */
					$values = get_post_meta( $the_post->ID, $field, false );
				}
			}
			$values = is_array($values) ? $values : array( $values );
			$values = apply_filters( 'asp_post_custom_field_before_tokenize', $values, $the_post, $field, $tokens );

			/**
			 * For ACF fields, we also need to index the labels if present (checkbox, radio etc...)
			 */
			$acf_labels = array();
			if ( function_exists( 'get_field_object') ) {
				$acf_labels = asp_acf_get_field_choices($field);
			}

			foreach ( $values as $value ) {
				if ( is_string($value) && Str::isJson($value) ) {
					$value = json_decode($value, true);
				}

				$value = !is_array($value) ? array( $value ) : $value;

				foreach ( $value as $v ) {
					if ( $field_type === 'user' ) {
						if ( is_numeric($v) ) {
							$user = get_userdata( $v );
							if ( !is_wp_error($user) && isset($user->display_name) ) {
								$cf_content .= ' ' . $user->display_name;
							}
						}
					} elseif ( $field_type === 'post_object' || $field_type === 'relationship' ) {
						$post_id = is_numeric($v) ? intval($v) : ( is_object($v) && isset($v->ID) ? $v->ID : null );
						if ( $post_id ) {
							$reference_post_title = get_the_title($v);
							if ( !is_wp_error($reference_post_title) && $reference_post_title !== '' ) {
								$cf_content .= ' ' . $reference_post_title;
							}
						}
					} else {
						if ( is_object($v) ) { // In case of objects try fetching the IDs
							if ( isset($v->ID) ) {
								$title = get_the_title($v->ID);
								if ( !is_wp_error($title) && $title !== '' ) {
									$v = $title;
								}
							}
						}
						$v = Str::anyToString( $v );
						if ( $v !== '' ) {
							if ( isset($acf_labels[ $v ]) && $v != $acf_labels[ $v ] ) {
								$v .= ' ' . $acf_labels[ $v ];
							}
							$cf_content .= ' ' . $v;
							// Without spaces for short values (for example product SKUs)
							if ( $fn_strlen($v) <= 50 ) {
								$spaceless_value = str_replace(' ', '', $v);
								if ( $spaceless_value !== $v ) {
									$cf_content .= ' ' . $spaceless_value;
								}
							}
						}
					}
				}
			}

			if ( count($values) == 1 ) {

				/**
				 * For numeric values where fields end with "_id" or "ID", try to fetch a reference post
				 */
				if ( empty($acf_labels) ) {
					foreach ( $values as $value ) {
						if (
							is_numeric($value) &&
							( substr($field, -strlen('ID')) ==='ID' || substr($field, -strlen('_id')) ==='_id' )
						) {
							$reference_post_title = get_the_title($value);
							if ( !is_wp_error($reference_post_title) && $reference_post_title != '' ) {
								$cf_content .= ' ' . $reference_post_title;
							}
						}
					}
				}

				foreach ( $values as $value ) {
					// No-reverse exact field
					$single_cf_content = $this->tokenizer->tokenizeSimple($value, $the_post);
					if ( $single_cf_content != '' ) {
						$single_cf_content = MB::substr($single_cf_content, 0, 45);
						$pos               = MB::strpos($single_cf_content, ' ');

						/*
						 * The index table unique key is (doc, term, item) - so if this word already exists, then it will be ignored
						 * by the database. To make sure it is added, append a unique string at the end, that is not searched.
						 */
						if ( $pos === false ) {
							$single_cf_content .= '___';
						}
						$this->insertToken($tokens, $single_cf_content, 1, 'customfield', true);

						$single_cf_content_al = str_replace(self::$apostrophes, '', $single_cf_content);
						if ( $single_cf_content_al !== $single_cf_content ) {
							$this->insertToken($tokens, $single_cf_content_al, 1, 'customfield', true);
						}
					}
				}
			}
		}

		$cf_content = apply_filters('asp_index_cf_contents_before_tokenize', $cf_content, $the_post);

		if ( $cf_content != '' ) {
			$cf_keywords = $this->tokenizer->tokenize( $cf_content, $the_post, $this->lang );
			foreach ( $cf_keywords as $keyword ) {
				$this->insertToken( $tokens, $keyword[0], $keyword[1], 'customfield' );
			}

			return count( $cf_keywords );
		}

		return 0;
	}


	/**
	 * Puts the keyword token into the tokens array.
	 *
	 * @param array  $tokens array to the tokens
	 * @param string $keyword keyword
	 * @param int    $count keyword occurrence count
	 * @param string $field the field
	 * @param bool   $no_reverse if the reverse keyword should be stored
	 */
	private function insertToken( array &$tokens, string $keyword, int $count = 1, string $field = 'content', bool $no_reverse = false ): void {
		// Take care of accidental empty keyowrds
		if ( trim($keyword) == '' ) {
			return;
		}
		if ( MB::strlen($keyword) > 50 ) {
			$keyword = MB::substr($keyword, 0, 50);
		}
		// Can't use numeric keys, it would break things...
		// We need to trim it at inserting
		$key = $keyword;
		if ( is_numeric( $keyword ) ) {
			$key = ' ' . $keyword;
		}

		// Preserve the non-reverse key uniqueness
		if ( $no_reverse ) {
			$key .= '__NOREV__';
		}

		if ( !isset($tokens[ $key ]) ) {
			$tokens[ $key ] = array(
				'content'     => 0,
				'title'       => 0,
				'comment'     => 0,
				'tag'         => 0,
				'link'        => 0,
				'author'      => 0,
				'excerpt'     => 0,
				'customfield' => 0,
				'_keyword'    => $keyword,
				'_no_reverse' => $no_reverse,
			);
		}
		$tokens[ $key ][ $field ] += $count;
	}

	/**
	 * A working hack to get the post language by post object WPML
	 *
	 * @param $post
	 *
	 * @return string language string
	 */
	private function wpml_langcode_post_id( $post ): string {
		global $wpdb;

		$post_type = 'post_' . $post->post_type;

		$query      = $wpdb->prepare(
			'
			SELECT language_code
			FROM ' . $wpdb->prefix . "icl_translations
			WHERE
			element_type = '%s' AND
			element_id = %d",
			$post_type,
			$post->ID
		);
		$query_exec = $wpdb->get_row($query);

		if ( null !== $query_exec ) {
			return $query_exec->language_code;
		}

		return '';
	}

	public function getPostIdsToIndexCount( bool $check_only = false ): int {
		$cursor = empty( $this->args['extend'] ) ? (int) get_option( 'asp_index_cursor', 0 ) : 0;
		return IndexTable::getPostIdsToIndexCount( $this->args, $check_only, $cursor );
	}


	/**
	 * Gets the number of so far indexed documents
	 *
	 * @return int number of indexed documents
	 * @noinspection PhpUnused
	 */
	public function getPostsIndexed(): int {
		return IndexTable::getPostsIndexed();
	}

	/**
	 * Gets the number of items in the index table, multisite supported
	 *
	 * @return int number of rows
	 */
	public function getTotalKeywords(): int {
		return IndexTable::getTotalKeywords();
	}

	public function isEmpty(): bool {
		return IndexTable::isEmpty();
	}

	/**
	 * Gets the number of indexed documents on this run instance
	 *
	 * @return int number of indexed documents
	 */
	private function getPostsIndexedNow(): int {
		return $this->posts_indexed_now;
	}
}
