diff --git a/lib/LANraragi/Controller/Api/Search.pm b/lib/LANraragi/Controller/Api/Search.pm index 170a757a8..2a3432eef 100644 --- a/lib/LANraragi/Controller/Api/Search.pm +++ b/lib/LANraragi/Controller/Api/Search.pm @@ -76,7 +76,7 @@ sub handle_api { my $filter = $req->param('filter'); my $category = $req->param('category') || ""; my $start = $req->param('start') || 0; - my $sortkey = $req->param('sortby'); + my $sortkey = $req->param('sortby') || "title"; my $sortorder = $req->param('order'); my $newfilter = $req->param('newonly') // "false"; my $untaggedf = $req->param('untaggedonly') // "false"; @@ -107,7 +107,7 @@ sub handle_api { } } -# Search endpoint returning only archive IDs for a query. +# handle_api but only return archive IDs sub handle_api_ids { my $self = shift->openapi->valid_input or return; @@ -116,7 +116,7 @@ sub handle_api_ids { my $filter = $req->param('filter'); my $category = $req->param('category') || ""; my $start = $req->param('start') || 0; - my $sortkey = $req->param('sortby'); + my $sortkey = $req->param('sortby') || "title"; my $sortorder = $req->param('order'); my $newfilter = $req->param('newonly') // "false"; my $untaggedf = $req->param('untaggedonly') // "false"; @@ -153,6 +153,76 @@ sub handle_api_ids { } } +# Composite search API with multi-clause OR support. +sub handle_composite { + + my $self = shift->openapi->valid_input or return; + my $body = $self->req->json; + + my $clauses_raw = $body->{clauses}; + my $start = $body->{start} // 0; + my $sortkey = $body->{sortby} || "title"; + my $sortorder = ( $body->{order} && $body->{order} eq 'desc' ) ? 1 : 0; + my $grouptanks = $body->{groupby_tanks} // 1; + + my ( $total, $filtered, @ids ) = LANraragi::Model::Search::do_composite_search( + $clauses_raw, $start, $sortkey, $sortorder, $grouptanks ? 1 : 0 + ); + + if ( $total eq -1 && $filtered eq -1 ) { + + # Search engine not initialized + $self->render( + openapi => { + recordsTotal => 0, + recordsFiltered => 0, + data => [] + }, + status => 204 + ); + } else { + $self->render( openapi => get_api_object( $total, $filtered, @ids ) ); + } +} + +# handle_composite but only return archive IDs +sub handle_composite_ids { + + my $self = shift->openapi->valid_input or return; + my $body = $self->req->json; + + my $clauses_raw = $body->{clauses}; + my $start = $body->{start} // 0; + my $sortkey = $body->{sortby} || "title"; + my $sortorder = ( $body->{order} && $body->{order} eq 'desc' ) ? 1 : 0; + my $grouptanks = $body->{groupby_tanks} // 1; + + my ( $total, $filtered, @ids ) = LANraragi::Model::Search::do_composite_search( + $clauses_raw, $start, $sortkey, $sortorder, $grouptanks ? 1 : 0 + ); + + if ( $total eq -1 && $filtered eq -1 ) { + + # Search engine not initialized + $self->render( + openapi => { + recordsTotal => 0, + recordsFiltered => 0, + data => [] + }, + status => 204 + ); + } else { + $self->render( + openapi => { + recordsTotal => $total, + recordsFiltered => $filtered, + data => \@ids + } + ); + } +} + sub clear_cache { invalidate_cache(); render_api_response( shift, "clear_cache" ); @@ -180,6 +250,44 @@ sub get_random_archives { $grouptanks eq "true", $hidecompleted eq "true" ); + + render_random_archives( $self, $random_count, @ids ); +} + +# Pull random archives out of a given composite search +sub get_random_archives_composite { + + my $self = shift->openapi->valid_input or return; + my $body = $self->req->json; + + my $clauses_raw = $body->{clauses}; + my $grouptanks = $body->{groupby_tanks} // 1; + my $random_count = $body->{count} // 5; + + # Use the search engine to get IDs matching the clause set, with start=-1 to get all data + my ( $total, $filtered, @ids ) = LANraragi::Model::Search::do_composite_search( + $clauses_raw, -1, "title", 0, $grouptanks ? 1 : 0 + ); + + if ( $total eq -1 && $filtered eq -1 ) { + + # Search engine not initialized + $self->render( + openapi => { + data => [], + recordsTotal => 0 + }, + status => 204 + ); + return; + } + + render_random_archives( $self, $random_count, @ids ); +} + +# Render up to $random_count archives picked at random out of @ids. +sub render_random_archives ( $self, $random_count, @ids ) { + my @random_ids; $random_count = min( $random_count, scalar(@ids) ); diff --git a/lib/LANraragi/Model/Search.pm b/lib/LANraragi/Model/Search.pm index fccf688a0..2a9d6c5c8 100644 --- a/lib/LANraragi/Model/Search.pm +++ b/lib/LANraragi/Model/Search.pm @@ -14,10 +14,10 @@ use Sort::Naturally; use Cpanel::JSON::XS qw(decode_json); use Time::HiRes qw(time); -use LANraragi::Utils::Generic qw(intersect_arrays); -use LANraragi::Utils::String qw(trim); -use LANraragi::Utils::Redis qw(redis_decode redis_encode); -use LANraragi::Utils::Logging qw(get_logger); +use LANraragi::Utils::Generic qw(intersect_arrays); +use LANraragi::Utils::Search qw(normalize_clauses reduce_clauses compute_search_filter resolve_search_clause); +use LANraragi::Utils::Redis qw(redis_decode redis_encode); +use LANraragi::Utils::Logging qw(get_logger); use LANraragi::Model::Archive; use LANraragi::Model::Category; @@ -27,13 +27,16 @@ use LANraragi::Model::Tankoubon qw(tank_has_archive_in_set get_tank_unified_tags # Performs a search on the database. sub do_search ( $filter, $category_id, $start, $sortkey, $sortorder, $newonly, $untaggedonly, $grouptanks, $hidecompleted ) { - my $redis = LANraragi::Model::Config->get_redis_search; - my $logger = get_logger( "Search Engine", "lanraragi" ); + my $redis = LANraragi::Model::Config->get_redis_search; + my $redis_db = LANraragi::Model::Config->get_redis; + my $logger = get_logger( "Search Engine", "lanraragi" ); unless ( $redis->exists("LAST_JOB_TIME") ) { $logger->warn("Search engine is not initialized yet. Please wait a few seconds."); # TODO - This is the only case where the API returns -1, but it's not really handled well clientside at the moment. + $redis->quit(); + $redis_db->quit(); return ( -1, -1, () ); } @@ -41,7 +44,7 @@ sub do_search ( $filter, $category_id, $start, $sortkey, $sortorder, $newonly, $ my $tankcount = $redis->scard("LRR_TANKGROUPED") + 0; # Get tank ids count - my $tankidscount = scalar( LANraragi::Model::Config->get_redis->keys('TANK_??????????') ); + my $tankidscount = scalar( $redis_db->keys('TANK_??????????') ); # Total number of archives (as int) my $total = $grouptanks ? $tankcount : $redis->zcard("LRR_TITLES") - $tankidscount; @@ -56,14 +59,36 @@ sub do_search ( $filter, $category_id, $start, $sortkey, $sortorder, $newonly, $ # Don't use cache for history searches since setting lastreadtime doesn't (and shouldn't) cachebust unless ( $cachehit && $sortkey ne "lastread" ) { $logger->debug("No cache available (or history-sorted search), doing a full DB parse."); + + # Resolve candidate set based on grouptanks mode + my @candidates; + if ($grouptanks) { + @candidates = $redis->smembers("LRR_TANKGROUPED"); + } else { + @candidates = $redis_db->keys('????????????????????????????????????????'); + } + + # Convert single category_id to structured format for resolve_clause_categories + my @categories = (); + if ( $category_id && $category_id ne "" ) { + push @categories, { id => $category_id, mode => "include" }; + } + + my @tokens = compute_search_filter( $filter // "" ); + my ( $cat_candidates, $extra_tokens, $exclude_ids ) = + resolve_clause_categories( $redis, $redis_db, \@categories, \@candidates ); + my $clause = resolve_search_clause( [ @tokens, @$extra_tokens ], + $exclude_ids, $cat_candidates, $newonly, $untaggedonly, $hidecompleted ); + my $keyed_count; - ( $keyed_count, @filtered ) = - search_uncached( $category_id, $filter, $sortkey, $sortorder, $newonly, $untaggedonly, $grouptanks, $hidecompleted ); + ( $keyed_count, @filtered ) = do_composite_search_inner( $redis, $redis_db, [$clause], $sortkey, $sortorder ); # Cache this query in the search database, prepending the keyed count for partition-aware cache inversion eval { $redis->hset( "LRR_SEARCHCACHE", $cachekey, nfreeze [ $keyed_count, @filtered ] ); }; } + $redis->quit(); + $redis_db->quit(); # If start is negative, return all possible data. if ( $start == -1 ) { @@ -78,6 +103,152 @@ sub do_search ( $filter, $category_id, $start, $sortkey, $sortorder, $newonly, $ return ( $total, $#filtered + 1, @filtered[ $start .. $end ] ); } +# do_composite_search (clause_descriptors, start, sortkey, sortorder, grouptanks) +# Performs a composite search: each clause descriptor is resolved into an AND conjunction, +# then multiple clauses are OR-unioned. Superset of do_search. +# +# Sort and pagination are global, applied after the OR union across all clauses. +# No caching for composite queries. +# +# Parameters: +# $clause_descriptors - arrayref of descriptor hashrefs, each containing: +# filter => search filter string +# categories => arrayref of { id, mode } hashrefs +# mode may be "include" or "exclude". +# newonly => 1 = only, -1 = exclude, 0 = off +# untaggedonly => 1 = only, -1 = exclude, 0 = off +# $start - pagination offset (-1 for all results) +# $sortkey - sort field: "title", "lastread", or a tag namespace +# $sortorder - 0 = ascending, 1 = descending +# $grouptanks - 0|1, determines candidate pool +# +# Returns: ($total, $filtered_count, @page_of_ids) +sub do_composite_search ( $clause_descriptors, $start, $sortkey, $sortorder, $grouptanks ) { + + my $redis = LANraragi::Model::Config->get_redis_search; + my $redis_db = LANraragi::Model::Config->get_redis; + my $logger = get_logger( "Search Engine", "lanraragi" ); + + unless ( $redis->exists("LAST_JOB_TIME") ) { + $logger->error("Search engine is not initialized yet. Please wait a few seconds."); + $redis->quit(); + $redis_db->quit(); + return ( -1, -1, () ); + } + + # Normalize once, reduce via DNF absorption, then resolve. + my $normed = normalize_clauses($clause_descriptors); + $normed = reduce_clauses($normed); + + my $tankcount = $redis->scard("LRR_TANKGROUPED") + 0; + my $tankidscount = scalar( $redis_db->keys('TANK_??????????') ); + my $total = $grouptanks ? $tankcount : $redis->zcard("LRR_TITLES") - $tankidscount; + + # Resolve base candidates from grouptanks mode + my @base_candidates; + if ($grouptanks) { + @base_candidates = $redis->smembers("LRR_TANKGROUPED"); + } else { + @base_candidates = $redis_db->keys('????????????????????????????????????????'); + } + + # Resolve each normalized clause + my @clauses; + foreach my $n (@$normed) { + my ( $candidates, $extra_tokens, $exclude_ids ) = + resolve_clause_categories( $redis, $redis_db, $n->{raw_categories}, \@base_candidates ); + push @clauses, resolve_search_clause( + [ @{ $n->{raw_tokens} }, @$extra_tokens ], + $exclude_ids, + $candidates, + $n->{newonly}, + $n->{untaggedonly}, + $n->{hidecompleted}, + ); + } + + my ( $keyed_count, @filtered ) = do_composite_search_inner( $redis, $redis_db, \@clauses, $sortkey, $sortorder ); + + $redis->quit(); + $redis_db->quit(); + + # If start is negative, return all possible data. + if ( $start == -1 ) { + return ( $total, $#filtered + 1, @filtered ); + } + + # Only get the first X keys + my $keysperpage = LANraragi::Model::Config->get_pagesize; + + my $end = min( $start + $keysperpage - 1, $#filtered ); + return ( $total, $#filtered + 1, @filtered[ $start .. $end ] ); +} + +# do_composite_search_inner (redis, redis_db, clauses, sortkey, sortorder) +# Core composite search logic. Runs search_core per clause, unions results, sorts globally. +# Accepts Redis connections from the caller. +# +# Parameters: +# $redis - Redis connection for search database +# $redis_db - Redis connection for main database +# $clauses - arrayref of clause hashrefs (see do_composite_search) +# $sortkey - sort field +# $sortorder - 0 = ascending, 1 = descending +# +# Returns: ($keyed_count, @sorted_ids) +sub do_composite_search_inner ( $redis, $redis_db, $clauses, $sortkey, $sortorder ) { + + # Run search_core per clause and union the membership results. + my @union; + if ( scalar @$clauses == 1 ) { + my $clause = $clauses->[0]; + @union = search_core( + $redis, $redis_db, + $clause->{candidate_ids}, $clause->{exclude_ids}, $clause->{tokens}, + $clause->{newonly}, $clause->{untaggedonly}, + $clause->{hidecompleted} + ); + } else { + my %seen; + foreach my $clause (@$clauses) { + my @results = search_core( + $redis, $redis_db, + $clause->{candidate_ids}, $clause->{exclude_ids}, $clause->{tokens}, + $clause->{newonly}, $clause->{untaggedonly}, + $clause->{hidecompleted} + ); + + # Deduplicate: preserve first occurrence across clauses + foreach my $id (@results) { + unless ( $seen{$id}++ ) { + push @union, $id; + } + } + } + } + + # Re-sort the union globally + if ( scalar @union > 0 ) { + if ( !$sortkey ) { + $sortkey = "title"; + } + + if ( $sortkey eq "title" ) { + my @ordered = nsort( $redis->zrangebylex( "LRR_TITLES", "-", "+" ) ); + if ($sortorder) { + @ordered = reverse(@ordered); + } + @ordered = map { substr( $_, index( $_, "\x00" ) + 1 ) } @ordered; + @union = intersect_arrays( \@union, \@ordered, 0 ); + return ( -1, @union ); + } else { + return sort_results( $sortkey, $sortorder, @union ); + } + } + + return ( -1, @union ); +} + sub check_cache ( $cachekey, $cachekey_inv ) { my $redis = LANraragi::Model::Config->get_redis_search; @@ -116,55 +287,48 @@ sub check_cache ( $cachekey, $cachekey_inv ) { return ( $cachehit, @filtered ); } -# Grab all our IDs, then filter them down according to the following filters and tokens' ID groups. -sub search_uncached ( $category_id, $filter, $sortkey, $sortorder, $newonly, $untaggedonly, $grouptanks, $hidecompleted ) { - - my $redis = LANraragi::Model::Config->get_redis_search; - my $redis_db = LANraragi::Model::Config->get_redis; - my $logger = get_logger( "Search Core", "lanraragi" ); - - # Compute search filters - my @tokens = compute_search_filter($filter); - - # Prepare array: For each token, we'll have a list of matching archive IDs. - # We intersect those lists as we proceed to get the final result. - my @filtered; - if ($grouptanks) { - - # Start with our tank IDs, and all other archive IDs that aren't in tanks - @filtered = $redis->smembers("LRR_TANKGROUPED"); - } else { - - # Start with all our archive IDs. Tank IDs won't be present in this search. - @filtered = $redis_db->keys('????????????????????????????????????????'); - } +# search_core (redis, redis_db, candidate_ids, exclude_ids, tokens, newonly, untaggedonly, hidecompleted) +# Core search function operating on a pre-resolved candidate set. Does not handle sorting. +# No category or grouptanks awareness — the caller resolves those into candidate_ids and tokens. +# +# Parameters: +# $redis - Redis connection for search database (indexes, titles, cache) +# $redis_db - Redis connection for main database (archive data) +# $candidate_ids - arrayref of IDs to search within (archive and/or tank IDs) +# $exclude_ids - arrayref of IDs to exclude from the final result (archive and/or tank IDs) +# $tokens - arrayref of token hashrefs from compute_search_filter, each { tag, isneg, isexact } +# $newonly - tri-state: 1 = only new, -1 = exclude new, 0 = off +# $untaggedonly - tri-state: 1 = only untagged, -1 = exclude untagged, 0 = off +# $hidecompleted - boolean: 1 = hide archives with progress/pagecount > 0.85 +# +# Returns: the IDs matching the clause. +sub search_core ( $redis, $redis_db, $candidate_ids, $exclude_ids, $tokens, $newonly, $untaggedonly, $hidecompleted ) { - # If we're using a category, we'll need to get its source data first. - my %category = LANraragi::Model::Category::get_category($category_id); + my $logger = get_logger( "Search Core", "lanraragi" ); - if (%category) { + my @filtered = @$candidate_ids; - # If the category is dynamic, get its search predicate and add it to the tokens. - # If it's static however, we can use its ID list as the base for our result array. - if ( $category{search} ne "" ) { - my @cat_tokens = compute_search_filter( $category{search} ); - push @tokens, @cat_tokens; - } else { - @filtered = intersect_arrays( $category{archives}, \@filtered, 0 ); - } + # Empty candidate set: no results possible + if ( scalar @filtered == 0 ) { + return (); } - # If the untagged filter is enabled, call the untagged files API + # Untagged filter: 1 = only untagged, -1 = only tagged if ($untaggedonly) { my @untagged = $redis->smembers("LRR_UNTAGGED"); - @filtered = intersect_arrays( \@untagged, \@filtered, 0 ); + my $isneg = ( $untaggedonly == -1 ) ? 1 : 0; + @filtered = intersect_arrays( \@untagged, \@filtered, $isneg ); } # Check new filter if ($newonly) { my @new = $redis->smembers("LRR_NEW"); my %new_set = map { $_ => 1 } @new; - @filtered = grep { /^TANK/ ? tank_has_archive_in_set( $_, \%new_set ) : $new_set{$_} } @filtered; + my $isneg = ( $newonly == -1 ) ? 1 : 0; + @filtered = grep { + my $in = /^TANK/ ? tank_has_archive_in_set( $_, \%new_set ) : $new_set{$_}; + $isneg ? !$in : $in; + } @filtered; } # Hide completed archives (Consider an archive read if progress is past 85 % of total) @@ -221,8 +385,8 @@ LUA } # Iterate through each token and intersect the results with the previous ones. - unless ( scalar @tokens == 0 || scalar @filtered == 0 ) { - foreach my $token (@tokens) { + unless ( scalar @$tokens == 0 || scalar @filtered == 0 ) { + foreach my $token (@$tokens) { my $tag = $token->{tag}; my $isneg = $token->{isneg}; @@ -342,135 +506,50 @@ LUA } if ( scalar @filtered > 0 ) { - $logger->debug( "Found " . scalar @filtered . " results after filtering." ); - - if ( !$sortkey ) { - $sortkey = "title"; + # Static category exclusions + if ( $exclude_ids && @$exclude_ids ) { + @filtered = intersect_arrays( $exclude_ids, \@filtered, 1 ); + return () unless @filtered; } - if ( $sortkey eq "title" ) { - my @ordered = (); - - # For title sorting, we can just use the LRR_TITLES set, which is sorted lexicographically (but not naturally). - @ordered = nsort( $redis->zrangebylex( "LRR_TITLES", "-", "+" ) ); - if ($sortorder) { - @ordered = reverse(@ordered); - } - - # Remove the titles from the keys, which are stored as "title\x00id" - @ordered = map { substr( $_, index( $_, "\x00" ) + 1 ) } @ordered; - - $logger->trace( "Example element from ordered list: " . $ordered[0] ); - - # Just intersect the ordered list with the filtered one to get the final result - @filtered = intersect_arrays( \@filtered, \@ordered, 0 ); - } else { - - # For other sorting, we need to get the metadata for each archive and sort it manually. - my $keyed_count; - ( $keyed_count, @filtered ) = sort_results( $sortkey, $sortorder, @filtered ); - - $redis->quit(); - $redis_db->quit(); - return ( $keyed_count, @filtered ); - } + $logger->debug( "Found " . scalar @filtered . " results after filtering." ); } - $redis->quit(); - $redis_db->quit(); - - # Title sort and unfiltered results: all archives are keyed - return ( -1, @filtered ); + return @filtered; } -# Transform the search engine syntax into a list of tokens. -# A token object contains the tag, whether it must be an exact match, and whether it must be absent. -sub compute_search_filter ($filter) { - - my $logger = get_logger( "Search Core", "lanraragi" ); - my @tokens = (); - if ( !$filter ) { $filter = ""; } - - # Special characters: - # "" for exact search (or $, but is that one really useful now?) - # ?/_ for any character - # * % for multiple characters - # - to exclude the next tag - - $b = reverse($filter); - while ( $b ne "" ) { - - my $char = chop $b; - my $isneg = 0; - - # Skip spaces - while ( $char eq " " && $b ne "" ) { - $char = chop $b; - } - - if ( $char eq "-" ) { - $isneg = 1; - $char = chop $b; - } - - # Get characters until the next comma, or the next " if the following char is " - my $delimiter = ','; - if ( $char eq '"' ) { - $delimiter = '"'; - $char = chop $b; - } - - my $tag = ""; - my $isexact = 0; - TAGBUILD: while (1) { - if ( $char eq $delimiter || $char eq "" ) { last TAGBUILD; } - $tag = $tag . $char; # Add characters in reverse order since we used reverse earlier on - $char = chop $b; +# Filter candidates through included static or excluded dynamic categories, and collect +# included dynamic tokens and excluded static archive ids for resolve_search_clause. +# Returns: (\@candidates, \@extra_tokens, \@exclude_ids) +sub resolve_clause_categories ( $redis, $redis_db, $categories, $base_candidates ) { + my @candidates = @$base_candidates; + my @extra_tokens = (); + my @exclude_ids = (); + return ( \@candidates, \@extra_tokens, \@exclude_ids ) unless $categories && @$categories; + + foreach my $cat_entry (@$categories) { + last unless @candidates; + my %category = LANraragi::Model::Category::get_category( $cat_entry->{id} ); + next unless %category; + my $mode = $cat_entry->{mode} // "include"; + + if ( $mode eq "include" && $category{search} eq "" ) { + # include static category + @candidates = intersect_arrays( $category{archives}, \@candidates, 0 ); + } elsif ( $mode eq "exclude" && $category{search} ne "" ) { + # exclude dynamic category + my @tokens = compute_search_filter( $category{search} ); + my @members = search_core( $redis, $redis_db, \@candidates, [], \@tokens, 0, 0, 0 ); + @candidates = intersect_arrays( \@members, \@candidates, 1 ); + } elsif ( $mode eq "include" ) { + # include dynamic category + push @extra_tokens, compute_search_filter( $category{search} ); + } elsif ( $mode eq "exclude" ) { + # exclude static category + push @exclude_ids, @{ $category{archives} }; } - - #If last char is $ or delimiter was ", enable isexact - if ( $delimiter eq '"' ) { - $isexact = 1; - - # Quotes then $ is an accepted syntax, even though it does nothing - $char = chop $b; - unless ( $char eq "\$" ) { - $b = $b . $char; - } - } else { - $char = chop $tag; - if ( $char eq "\$" ) { - $isexact = 1; - } else { - $tag = $tag . $char; - } - } - - # Escape already present regex characters - $logger->debug("Pre-escaped tag: $tag"); - - $tag = trim($tag); - - # Escape characters according to redis zscan rules - $tag =~ s/([\[\]\^\\])/\\$1/g; - - # Replace placeholders with glob-style patterns, - # ? or _ => ? - $tag =~ s/\_/\?/g; - - # * or % => * - $tag =~ s/\%/\*/g; - - if ( $tag ne "" ) { # Blank tokens shouldn't be added as theyll slow down search - push @tokens, - { tag => lc($tag), - isneg => $isneg, - isexact => $isexact - }; - } - } - return @tokens; + return ( \@candidates, \@extra_tokens, \@exclude_ids ); } sub sort_results ( $sortkey, $sortorder, @filtered ) { @@ -483,6 +562,7 @@ sub sort_results ( $sortkey, $sortorder, @filtered ) { # Should there be no IDs requiring sorting, return an empty array directly if ( scalar @filtered == 0 ) { + $redis->quit(); return ( 0, @sorted ); } @@ -622,6 +702,7 @@ LUA my $total_time = time() - $start_time; $logger->debug("[PERF] sort_results completed in ${total_time}s"); + $redis->quit(); return ( scalar @keyed_ids, @sorted ); } } @@ -684,4 +765,4 @@ sub _fallback_tags ( $redis, $tmpfilter, $re, @filtered ) { ); } -1; \ No newline at end of file +1; diff --git a/lib/LANraragi/Utils/Search.pm b/lib/LANraragi/Utils/Search.pm new file mode 100644 index 000000000..983f66114 --- /dev/null +++ b/lib/LANraragi/Utils/Search.pm @@ -0,0 +1,222 @@ +# Search algorithm assumption constraints for (some) optimization: +# +# - number of archives in static category << number of total archives +# - number of tokens in dynamic category is small +# +# This breaks our handling of cases to the following: +# +# 1. If an including category is dynamic, filter candidates after the search. +# 2. If an including category is static, filter candidates before the search. +# 3. If an excluding category is dynamic, exclude candidates before the search. +# 4. If an excluding category is static, exclude candidates after the search. +# +# "before the search" cases are thus evaluated before `resolve_search_clause`, and the results +# should be passed through as `base_candidates`. +package LANraragi::Utils::Search; + +use strict; +use warnings; +use utf8; +no warnings 'experimental'; +use feature qw(signatures); + +use Exporter 'import'; +our @EXPORT_OK = qw(reduce_clauses normalize_clauses compute_search_filter resolve_search_clause); + +use LANraragi::Utils::String qw(trim); +use LANraragi::Utils::Generic qw(intersect_arrays); +use LANraragi::Utils::Logging qw(get_logger); + +# compute_search_filter (filter) +# Transform the search engine syntax into a list of tokens. +# A token object contains the tag, whether it must be an exact match, and whether it must be absent. +sub compute_search_filter ($filter) { + + my $logger = get_logger( "Search Core", "lanraragi" ); + my @tokens = (); + if ( !$filter ) { $filter = ""; } + + # Special characters: + # "" for exact search (or $, but is that one really useful now?) + # ?/_ for any character + # * % for multiple characters + # - to exclude the next tag + + $b = reverse($filter); + while ( $b ne "" ) { + + my $char = chop $b; + my $isneg = 0; + + # Skip spaces + while ( $char eq " " && $b ne "" ) { + $char = chop $b; + } + + if ( $char eq "-" ) { + $isneg = 1; + $char = chop $b; + } + + # Get characters until the next comma, or the next " if the following char is " + my $delimiter = ','; + if ( $char eq '"' ) { + $delimiter = '"'; + $char = chop $b; + } + + my $tag = ""; + my $isexact = 0; + TAGBUILD: while (1) { + if ( $char eq $delimiter || $char eq "" ) { last TAGBUILD; } + $tag = $tag . $char; # Add characters in reverse order since we used reverse earlier on + $char = chop $b; + } + + #If last char is $ or delimiter was ", enable isexact + if ( $delimiter eq '"' ) { + $isexact = 1; + + # Quotes then $ is an accepted syntax, even though it does nothing + $char = chop $b; + unless ( $char eq "\$" ) { + $b = $b . $char; + } + } else { + $char = chop $tag; + if ( $char eq "\$" ) { + $isexact = 1; + } else { + $tag = $tag . $char; + } + } + + # Escape already present regex characters + $logger->debug("Pre-escaped tag: $tag"); + + $tag = trim($tag); + + # Escape characters according to redis zscan rules + $tag =~ s/([\[\]\^\\])/\\$1/g; + + # Replace placeholders with glob-style patterns, + # ? or _ => ? + $tag =~ s/\_/\?/g; + + # * or % => * + $tag =~ s/\%/\*/g; + + if ( $tag ne "" ) { # Blank tokens shouldn't be added as theyll slow down search + push @tokens, + { tag => lc($tag), + isneg => $isneg, + isexact => $isexact + }; + } + + } + return @tokens; +} + +# resolve_search_clause (tokens, exclude_ids, base_candidates, newonly, untaggedonly, hidecompleted) +# Resolves a search clause into a clause hashref for do_composite_search_inner. +# +# Returns: hashref { candidate_ids, tokens, newonly, untaggedonly, hidecompleted } +sub resolve_search_clause ( $tokens, $exclude_ids, $base_candidates, $newonly, $untaggedonly, $hidecompleted ) { + + return { + candidate_ids => $base_candidates, + exclude_ids => $exclude_ids, + tokens => $tokens, + newonly => $newonly, + untaggedonly => $untaggedonly, + hidecompleted => $hidecompleted, + }; +} + +# Parse each clause descriptor's filter and produce a normalized form for reduction and resolution. +sub normalize_clauses ($clause_descriptors) { + + my @normed; + for my $desc (@$clause_descriptors) { + my @tokens = compute_search_filter( $desc->{filter} ); + my @canon_tokens = sort map { join( "|", $_->{tag}, $_->{isneg}, $_->{isexact} ) } @tokens; + my $categories = $desc->{categories} // []; + my @canon_cats = sort map { $_->{id} . ":" . $_->{mode} } @$categories; + + push @normed, { + tokens => \@canon_tokens, + raw_tokens => \@tokens, + categories => \@canon_cats, + raw_categories => $categories, + newonly => $desc->{newonly} // 0, + untaggedonly => $desc->{untaggedonly} // 0, + hidecompleted => $desc->{hidecompleted} // 0, + }; + } + + return \@normed; +} + +# Reduce redundant normalized clauses via DNF absorption. +sub reduce_clauses ($normed) { + + return $normed if scalar @$normed <= 1; + + # Pairwise absorption: if A subsumes B, remove B. + # A subsumes B when A's predicates are a subset of B's (A is less restrictive). + # Identical clauses mutually subsume, so dedup is handled implicitly. + my @keep = (1) x scalar @$normed; + for my $i ( 0 .. $#$normed ) { + next unless $keep[$i]; + for my $j ( 0 .. $#$normed ) { + next if $i == $j; + next unless $keep[$j]; + + my ( $a, $b ) = ( $normed->[$i], $normed->[$j] ); + + # Flag subsumption: 0 (off) subsumes 1 (on) and -1 (exclude). + # Non-zero unequal values do not subsume (different filter semantics). + my $subsumes = 1; + for my $flag (qw(newonly untaggedonly hidecompleted)) { + unless ( $a->{$flag} == 0 || $a->{$flag} == $b->{$flag} ) { + $subsumes = 0; + last; + } + } + + # A's tokens must be a subset of B's tokens + if ($subsumes) { + my %b_tokens = map { $_ => 1 } @{ $b->{tokens} }; + for my $t ( @{ $a->{tokens} } ) { + unless ( $b_tokens{$t} ) { + $subsumes = 0; + last; + } + } + } + + # A's categories must be a subset of B's categories + if ($subsumes) { + my %b_cats = map { $_ => 1 } @{ $b->{categories} }; + for my $c ( @{ $a->{categories} } ) { + unless ( $b_cats{$c} ) { + $subsumes = 0; + last; + } + } + } + + $keep[$j] = 0 if $subsumes; + } + } + + my @result; + for my $i ( 0 .. $#$normed ) { + push @result, $normed->[$i] if $keep[$i]; + } + + return \@result; +} + +1; diff --git a/tests/LANraragi/Utils/Search.t b/tests/LANraragi/Utils/Search.t new file mode 100644 index 000000000..2a745aae4 --- /dev/null +++ b/tests/LANraragi/Utils/Search.t @@ -0,0 +1,203 @@ +use strict; +use warnings; +use utf8; +use Cwd; + +use Test::More; + +# Mock Redis +my $cwd = getcwd; +require $cwd . "/tests/mocks.pl"; +setup_redis_mock(); + +BEGIN { use_ok('LANraragi::Utils::Search'); } + +# Helper: normalize then reduce +sub norm_reduce { + my ($descs) = @_; + my $normed = LANraragi::Utils::Search::normalize_clauses($descs); + return LANraragi::Utils::Search::reduce_clauses($normed); +} + +note('testing reduce_clauses: single clause passthrough...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 1, 'Single clause should pass through unchanged' ); +} + +note('testing reduce_clauses: identical clause dedup...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 1, 'Identical clauses should deduplicate to 1' ); +} + +note('testing reduce_clauses: triple identical clause dedup...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 1, 'Three identical clauses should deduplicate to 1' ); +} + +note('testing reduce_clauses: distinct clauses preserved...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:shirow masamune", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 2, 'Distinct clauses should both be preserved' ); +} + +note('testing reduce_clauses: absorption A OR (A AND B) -> A...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:wada rco, character:ereshkigal", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 1, 'A OR (A AND B) should absorb to 1 clause' ); + is( scalar @{ $result->[0]{tokens} }, 1, 'Surviving clause should have 1 token (less restrictive)' ); +} + +note('testing reduce_clauses: absorption order independent...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco, character:ereshkigal", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 1, 'Absorption should work regardless of clause order' ); + is( scalar @{ $result->[0]{tokens} }, 1, 'Surviving clause should have 1 token (less restrictive)' ); +} + +note('testing reduce_clauses: irreducible clauses preserved...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco, character:ereshkigal", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "character:ereshkigal, language:english", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 2, 'Irreducible clauses should both be preserved' ); +} + +note('testing reduce_clauses: empty filter subsumes any filter...'); +{ + my $result = norm_reduce([ + { filter => "", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 1, 'Empty filter should absorb any filtered clause' ); + is( scalar @{ $result->[0]{tokens} }, 0, 'Surviving clause should have 0 tokens' ); +} + +note('testing reduce_clauses: newonly flag prevents absorption...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco", categories => [], newonly => 1, untaggedonly => 0 }, + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 1, 'newonly=0 should absorb newonly=1 with same filter' ); + is( $result->[0]{newonly}, 0, 'Surviving clause should have newonly=0' ); +} + +note('testing reduce_clauses: newonly=1 does not subsume newonly=-1...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco", categories => [], newonly => 1, untaggedonly => 0 }, + { filter => "artist:wada rco", categories => [], newonly => -1, untaggedonly => 0 }, + ]); + is( scalar @$result, 2, 'newonly=1 and newonly=-1 should not absorb each other' ); +} + +note('testing reduce_clauses: category subset absorption...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:wada rco", categories => [{ id => "SET_1589141306", mode => "include" }], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 1, 'Clause without categories should absorb clause with categories' ); + is( scalar @{ $result->[0]{categories} }, 0, 'Surviving clause should have no categories' ); +} + +note('testing reduce_clauses: category superset not absorbed...'); +{ + my $result = norm_reduce([ + { filter => "", categories => [{ id => "SET_1589141306", mode => "include" }], newonly => 0, untaggedonly => 0 }, + { filter => "", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 1, 'Empty clause should absorb category-filtered clause' ); + is( scalar @{ $result->[0]{categories} }, 0, 'Surviving clause should be the unconstrained one' ); +} + +note('testing reduce_clauses: multi-category absorption...'); +{ + my $result = norm_reduce([ + { filter => "", categories => [{ id => "SET_1589141306", mode => "include" }], newonly => 0, untaggedonly => 0 }, + { filter => "", categories => [{ id => "SET_1589141306", mode => "include" }, { id => "SET_1589138380", mode => "include" }], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 1, 'Single-category clause should absorb multi-category clause' ); + is( scalar @{ $result->[0]{categories} }, 1, 'Surviving clause should have 1 category' ); +} + +note('testing reduce_clauses: chain absorption A OR (A,B) OR (A,B,C)...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:wada rco, character:ereshkigal", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:wada rco, character:ereshkigal, language:english", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 1, 'Chain absorption should reduce to 1 clause' ); + is( scalar @{ $result->[0]{tokens} }, 1, 'Surviving clause should have 1 token (least restrictive)' ); +} + +note('testing reduce_clauses: mixed absorption and irreducible...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:wada rco, character:ereshkigal", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:shirow masamune", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 2, 'Mixed absorption should keep A and C, remove A,B' ); +} + +note('testing reduce_clauses: untaggedonly flag absorption...'); +{ + my $result = norm_reduce([ + { filter => "", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "", categories => [], newonly => 0, untaggedonly => 1 }, + ]); + is( scalar @$result, 1, 'untaggedonly=0 should absorb untaggedonly=1' ); + is( $result->[0]{untaggedonly}, 0, 'Surviving clause should have untaggedonly=0' ); +} + +note('testing reduce_clauses: combined filter+flag absorption...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "artist:wada rco, character:ereshkigal", categories => [], newonly => 1, untaggedonly => 0 }, + ]); + is( scalar @$result, 1, 'Less restrictive filter+flags should absorb more restrictive' ); + is( scalar @{ $result->[0]{tokens} }, 1, 'Surviving clause tokens' ); + is( $result->[0]{newonly}, 0, 'Surviving clause newonly' ); +} + +note('testing reduce_clauses: negated token asymmetry...'); +{ + my $result = norm_reduce([ + { filter => "artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + { filter => "-artist:wada rco", categories => [], newonly => 0, untaggedonly => 0 }, + ]); + is( scalar @$result, 2, 'Positive and negated token should be irreducible' ); +} + +note('testing reduce_clauses: empty input...'); +{ + my $result = LANraragi::Utils::Search::reduce_clauses([]); + is( scalar @$result, 0, 'Empty input should return empty' ); +} + +done_testing(); diff --git a/tests/modules.t b/tests/modules.t index 6d08329de..a345e56b5 100644 --- a/tests/modules.t +++ b/tests/modules.t @@ -16,7 +16,7 @@ my @modules = ( "LANraragi::Utils::Generic", "LANraragi::Utils::Plugins", "LANraragi::Utils::Routing", "LANraragi::Utils::TempFolder", "LANraragi::Utils::Logging", "LANraragi::Utils::OpenAPI", - "LANraragi::Utils::Minion", + "LANraragi::Utils::Minion", "LANraragi::Utils::Search", "LANraragi::Utils::Tags", "LANraragi::Controller::Api::Archive", "LANraragi::Controller::Api::Search", "LANraragi::Controller::Api::Category", "LANraragi::Controller::Api::Database", "LANraragi::Controller::Api::Shinobu", diff --git a/tests/search.t b/tests/search.t index 4d69f16e7..4bc135f8c 100644 --- a/tests/search.t +++ b/tests/search.t @@ -14,6 +14,7 @@ use Data::Dumper; use LANraragi::Model::Search; use LANraragi::Model::Config; use LANraragi::Model::Stats; +use LANraragi::Utils::Search; # Mock Redis my $cwd = getcwd; @@ -215,6 +216,418 @@ my %expected_unkeyed = map { $_ => 1 } ( 'Artist desc should keep all unkeyed archives at positions 5-13' ); } +# ── Composite search tests ────────────────────────────────────────────────── +# Tests call do_composite_search_inner directly with pre-resolved clauses. +# The mock redis is the same object for both $redis and $redis_db. + +my $redis_search = LANraragi::Model::Config->get_redis_search; +my $redis_db = LANraragi::Model::Config->get_redis; + +# All 9 archive IDs (non-grouptanks candidate set) +my @all_archive_ids = $redis_db->keys('????????????????????????????????????????'); + +note('testing composite search: single-clause equivalence...'); +{ + # Single clause matching do_search("artist:wada rco", "", 0, 0, 0, 0, 0, 0) + my @tokens = LANraragi::Utils::Search::compute_search_filter("artist:wada rco"); + my $clause = { + candidate_ids => \@all_archive_ids, + tokens => \@tokens, + newonly => 0, + untaggedonly => 0, + }; + + my ( $kc_composite, @composite_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause], 0, 0 ); + + my ( $total_ds, $filtered_ds, @ds_ids ) = + LANraragi::Model::Search::do_search( "artist:wada rco", "", -1, 0, 0, 0, 0, 0, 0 ); + + is( scalar @composite_ids, $filtered_ds, + 'Single-clause composite should match do_search result count' ); + is_deeply( \@composite_ids, \@ds_ids, + 'Single-clause composite should produce identical ID list to do_search' ); +} + +note('testing composite search: OR union of disjoint sets...'); +{ + my @tokens_a = LANraragi::Utils::Search::compute_search_filter("artist:wada rco"); + my @tokens_b = LANraragi::Utils::Search::compute_search_filter("artist:shirow masamune"); + + my $clause_a = { + candidate_ids => \@all_archive_ids, + tokens => \@tokens_a, + newonly => 0, + untaggedonly => 0, + }; + my $clause_b = { + candidate_ids => \@all_archive_ids, + tokens => \@tokens_b, + newonly => 0, + untaggedonly => 0, + }; + + my ( $kc, @union_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause_a, $clause_b], 0, 0 ); + + # wada rco: ...1eb3, ...22fd; shirow masamune: ...8630 + is( scalar @union_ids, 3, 'Disjoint OR union should contain 3 results' ); + + my %union_set = map { $_ => 1 } @union_ids; + ok( $union_set{"2810d5e0a8d027ecefebca6237031a0fa7b91eb3"}, 'OR union contains wada rco archive (Fate GO MEMO 2)' ); + ok( $union_set{"28697b96f0ac5858be2614ed10ca47742c9522fd"}, 'OR union contains wada rco archive (Fate GO MEMO)' ); + ok( $union_set{"4857fd2e7c00db8b0af0337b94055d8445118630"}, 'OR union contains shirow masamune archive' ); +} + +note('testing composite search: OR union with deduplication...'); +{ + # character:segata (fuzzy) matches ...ebf, ...ebg (+ title hits) + # male:very cool (fuzzy) matches ...ebf, ...22fd (+ title hits) + # ...ebf overlaps + my @tokens_a = LANraragi::Utils::Search::compute_search_filter("character:segata"); + my @tokens_b = LANraragi::Utils::Search::compute_search_filter("male:very cool"); + + my $clause_a = { + candidate_ids => \@all_archive_ids, + tokens => \@tokens_a, + newonly => 0, + untaggedonly => 0, + }; + my $clause_b = { + candidate_ids => \@all_archive_ids, + tokens => \@tokens_b, + newonly => 0, + untaggedonly => 0, + }; + + my ( $kc, @union_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause_a, $clause_b], 0, 0 ); + + # Count occurrences of each ID to verify no duplicates + my %id_counts; + $id_counts{$_}++ for @union_ids; + my @dupes = grep { $id_counts{$_} > 1 } keys %id_counts; + is( scalar @dupes, 0, 'OR union should contain no duplicate IDs' ); + + # ...ebf must be present (it's in both clauses) + ok( $id_counts{"e69e43e1355267f7d32a4f9b7f2fe108d2401ebf"}, + 'Overlapping archive appears exactly once in union' ); +} + +note('testing composite search: multi-category AND simulation...'); +{ + # Simulate: static "Segata Sanshiro" AND dynamic "AMERICA ONRY" + # Static category provides candidate_ids = [...ebf, ...ebg] + # Dynamic category provides tokens = compute_search_filter("American") + my @static_ids = ( + "e69e43e1355267f7d32a4f9b7f2fe108d2401ebf", + "e69e43e1355267f7d32a4f9b7f2fe108d2401ebg", + ); + my @tokens = LANraragi::Utils::Search::compute_search_filter("American"); + + my $clause = { + candidate_ids => \@static_ids, + tokens => \@tokens, + newonly => 0, + untaggedonly => 0, + }; + + my ( $kc, @result_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause], 0, 0 ); + + is( scalar @result_ids, 1, 'Multi-category AND should return 1 result' ); + is( $result_ids[0], "e69e43e1355267f7d32a4f9b7f2fe108d2401ebg", + 'Multi-category AND should return Saturn US Manual' ); +} + +note('testing composite search: category NOT simulation...'); +{ + # Exclude static "Segata Sanshiro" archives from candidate set + my %excluded = map { $_ => 1 } ( + "e69e43e1355267f7d32a4f9b7f2fe108d2401ebf", + "e69e43e1355267f7d32a4f9b7f2fe108d2401ebg", + ); + my @candidates = grep { !$excluded{$_} } @all_archive_ids; + my @tokens = (); + + my $clause = { + candidate_ids => \@candidates, + tokens => \@tokens, + newonly => 0, + untaggedonly => 0, + }; + + my ( $kc, @result_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause], 0, 0 ); + + is_deeply( [sort @result_ids], [sort grep { !$excluded{$_} } @all_archive_ids], + 'Category NOT should return all archives except excluded' ); +} + +note('testing composite search: per-clause newonly OR untaggedonly...'); +{ + # Clause A: newonly=1 → ...5f7e (Rohan) and ...955555 (All about Egypt), both isnew=true + # Clause B: untaggedonly=1 → ...8630 (Ghost in the Shell), ...5f7e (Rohan) + # Union should contain all three, with ...5f7e deduplicated + my $clause_a = { + candidate_ids => \@all_archive_ids, + tokens => [], + newonly => 1, + untaggedonly => 0, + }; + my $clause_b = { + candidate_ids => \@all_archive_ids, + tokens => [], + newonly => 0, + untaggedonly => 1, + }; + + my ( $kc, @union_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause_a, $clause_b], 0, 0 ); + + is_deeply( [sort @union_ids], + [sort ("e4c422fd10943dc169e3489a38cdbf57101a5f7e", "28697b96f0ac5858be2666ed10ca47742c955555", "4857fd2e7c00db8b0af0337b94055d8445118630")], + 'newonly OR untaggedonly should return both new archives and Ghost in the Shell' ); +} + +note('testing composite search: empty clause does not poison union...'); +{ + # Clause A: nonexistent tag → 0 results + # Clause B: artist:wada rco → 2 results + my @tokens_a = LANraragi::Utils::Search::compute_search_filter("nonexistent_tag_xyz_12345"); + my @tokens_b = LANraragi::Utils::Search::compute_search_filter("artist:wada rco"); + + my $clause_a = { + candidate_ids => \@all_archive_ids, + tokens => \@tokens_a, + newonly => 0, + untaggedonly => 0, + }; + my $clause_b = { + candidate_ids => \@all_archive_ids, + tokens => \@tokens_b, + newonly => 0, + untaggedonly => 0, + }; + + my ( $kc, @union_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause_a, $clause_b], 0, 0 ); + + is( scalar @union_ids, 2, 'Empty clause should not affect non-empty clause results' ); +} + +note('testing tri-state newonly: exclude new archives...'); +{ + my $clause = { + candidate_ids => \@all_archive_ids, + tokens => [], + newonly => -1, + untaggedonly => 0, + }; + + my ( $kc, @result_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause], 0, 0 ); + + is_deeply( [sort @result_ids], + [sort grep { $_ ne "e4c422fd10943dc169e3489a38cdbf57101a5f7e" && $_ ne "28697b96f0ac5858be2666ed10ca47742c955555" } @all_archive_ids], + 'Exclude new should return all archives except the two new ones' ); +} + +note('testing tri-state untaggedonly: exclude untagged archives...'); +{ + my $clause = { + candidate_ids => \@all_archive_ids, + tokens => [], + newonly => 0, + untaggedonly => -1, + }; + + my ( $kc, @result_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause], 0, 0 ); + + my @expected = sort grep { $_ ne "4857fd2e7c00db8b0af0337b94055d8445118630" && $_ ne "e4c422fd10943dc169e3489a38cdbf57101a5f7e" } @all_archive_ids; + is_deeply( [sort @result_ids], \@expected, + 'Exclude untagged should return only tagged archives' ); +} + +note('testing tri-state combined: tagged AND non-new...'); +{ + my $clause = { + candidate_ids => \@all_archive_ids, + tokens => [], + newonly => -1, + untaggedonly => -1, + }; + + my ( $kc, @result_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause], 0, 0 ); + + my @expected = sort grep { + $_ ne "4857fd2e7c00db8b0af0337b94055d8445118630" + && $_ ne "e4c422fd10943dc169e3489a38cdbf57101a5f7e" + && $_ ne "28697b96f0ac5858be2666ed10ca47742c955555" + } @all_archive_ids; + is_deeply( [sort @result_ids], \@expected, + 'Tagged AND non-new should exclude both new archives and Ghost in the Shell' ); +} + +note('testing composite search: tagged OR in-category...'); +{ + # Clause A: tagged archives (untaggedonly=-1) + # Clause B: archives in static category "Segata Sanshiro" (ebf, ebg) + # Union should be: all 7 tagged + ebf + ebg (but ebf/ebg are already tagged, so still 7) + my $clause_a = { + candidate_ids => \@all_archive_ids, + tokens => [], + newonly => 0, + untaggedonly => -1, + }; + + my @static_ids = ( + "e69e43e1355267f7d32a4f9b7f2fe108d2401ebf", + "e69e43e1355267f7d32a4f9b7f2fe108d2401ebg", + ); + my $clause_b = { + candidate_ids => \@static_ids, + tokens => [], + newonly => 0, + untaggedonly => 0, + }; + + my ( $kc, @union_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause_a, $clause_b], 0, 0 ); + + my @expected = sort grep { $_ ne "4857fd2e7c00db8b0af0337b94055d8445118630" && $_ ne "e4c422fd10943dc169e3489a38cdbf57101a5f7e" } @all_archive_ids; + is_deeply( [sort @union_ids], \@expected, + 'Tagged OR in-category should return all tagged archives' ); +} + +note('testing composite search: NOT category...'); +{ + # Exclude static "Segata Sanshiro" archives (ebf, ebg) from all archives + my %excluded = map { $_ => 1 } ( + "e69e43e1355267f7d32a4f9b7f2fe108d2401ebf", + "e69e43e1355267f7d32a4f9b7f2fe108d2401ebg", + ); + my @candidates = grep { !$excluded{$_} } @all_archive_ids; + + my $clause = { + candidate_ids => \@candidates, + tokens => [], + newonly => 0, + untaggedonly => 0, + }; + + my ( $kc, @result_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause], 0, 0 ); + + is_deeply( [sort @result_ids], [sort @candidates], + 'NOT category should return all archives except excluded' ); +} + +note('testing resolve_search_clause: category exclude...'); +{ + # Test resolve_search_clause with mode=exclude on static category "Segata Sanshiro" + my @tokens = LANraragi::Utils::Search::compute_search_filter(""); + my ( $candidates, $extra_tokens, $exclude_ids ) = LANraragi::Model::Search::resolve_clause_categories( + $redis_search, $redis_db, + [{ id => "SET_1589141306", mode => "exclude" }], + \@all_archive_ids + ); + my $clause = LANraragi::Utils::Search::resolve_search_clause( + [ @tokens, @$extra_tokens ], + $exclude_ids, + $candidates, 0, 0, 0 + ); + + my ( $kc, @result_ids ) = + LANraragi::Model::Search::do_composite_search_inner( $redis_search, $redis_db, [$clause], 0, 0 ); + + my @expected = sort grep { + $_ ne "e69e43e1355267f7d32a4f9b7f2fe108d2401ebf" && $_ ne "e69e43e1355267f7d32a4f9b7f2fe108d2401ebg" + } @all_archive_ids; + is_deeply( [sort @result_ids], \@expected, + 'resolve_search_clause exclude should return all archives except Saturn JP and US' ); +} + +# ── Category exclusion through the public composite entrypoint ─────────────── +# These drive do_composite_search with API-shaped clause descriptors, so they +# assert the endpoint contract rather than any particular internal wiring. + +my $CAT_STATIC = "SET_1589141306"; # static: Saturn JP + Saturn US +my $CAT_DYNAMIC = "SET_1589200000"; # dynamic, created below +my $SATURN_JP = "e69e43e1355267f7d32a4f9b7f2fe108d2401ebf"; +my $SATURN_US = "e69e43e1355267f7d32a4f9b7f2fe108d2401ebg"; +my $FATE_MEMO = "28697b96f0ac5858be2614ed10ca47742c9522fd"; # artist:wada rco only +my $FATE_MEMO_2 = "2810d5e0a8d027ecefebca6237031a0fa7b91eb3"; # artist:wada rco + character:ereshkigal + +sub composite_ids { + my (@clauses) = @_; + my ( undef, undef, @ids ) = LANraragi::Model::Search::do_composite_search( \@clauses, -1, "title", 0, 0 ); + return @ids; +} + +sub clause { + my ( $filter, $categories ) = @_; + return { + filter => $filter, + categories => $categories, + newonly => 0, + untaggedonly => 0, + hidecompleted => 0 + }; +} + +note('testing composite search: excluding a multi-token dynamic category...'); +{ + # Created here rather than in mocks.pl: opds.t asserts against a golden feed + # that enumerates every category as a facet link. + $redis_db->hset( $CAT_DYNAMIC, "id", $CAT_DYNAMIC ); + $redis_db->hset( $CAT_DYNAMIC, "name", "WADA X ERESHKIGAL" ); + $redis_db->hset( $CAT_DYNAMIC, "pinned", "0" ); + $redis_db->hset( $CAT_DYNAMIC, "archives", "[]" ); + $redis_db->hset( $CAT_DYNAMIC, "search", "artist:wada rco, character:ereshkigal" ); + + # Membership is a conjunction, so excluding the category must remove archives + # matching ALL its predicates -- not archives matching any one of them. + my @members = composite_ids( clause( "", [ { id => $CAT_DYNAMIC, mode => "include" } ] ) ); + is_deeply( \@members, [$FATE_MEMO_2], 'dynamic category contains only the archive matching both predicates' ); + + my @excluded = composite_ids( clause( "", [ { id => $CAT_DYNAMIC, mode => "exclude" } ] ) ); + + my %is_member = map { $_ => 1 } @members; + my @expected = sort grep { !$is_member{$_} } @all_archive_ids; + is_deeply( [ sort @excluded ], \@expected, 'excluding a dynamic category drops exactly its members' ); + + my %got = map { $_ => 1 } @excluded; + ok( $got{$FATE_MEMO}, 'archive matching only one category predicate is not a member, so it survives exclusion' ); + + $redis_db->del($CAT_DYNAMIC); +} + +note('testing composite search: excluding a static category...'); +{ + my @excluded = composite_ids( clause( "", [ { id => $CAT_STATIC, mode => "exclude" } ] ) ); + + my %is_member = map { $_ => 1 } ( $SATURN_JP, $SATURN_US ); + my @expected = sort grep { !$is_member{$_} } @all_archive_ids; + is_deeply( [ sort @excluded ], \@expected, 'excluding a static category drops exactly its members' ); +} + +note('testing composite search: a clause exclusion must not leak across the OR union...'); +{ + # Clause A excludes the static category; clause B independently matches one of + # its members. OR semantics require the union to keep it. + my @union = composite_ids( + clause( "", [ { id => $CAT_STATIC, mode => "exclude" } ] ), + clause( "American", [] ) + ); + + my %got = map { $_ => 1 } @union; + ok( $got{$SATURN_US}, 'archive excluded by one clause is retained when another clause matches it' ); + ok( !$got{$SATURN_JP}, 'archive excluded by one clause and matched by none is still dropped' ); +} + note('testing hidecompleted filter...'); # Completed archives in the mock data (progress / pagecount > 0.85): @@ -424,4 +837,21 @@ note('testing lastread sort with tanks (grouptanks=1) -- exercises _fallback_las $redis->hset( "28697b96f0ac5777be2614ed10ca47742c9522fa", "lastreadtime", 0 ); } +note('testing newonly tri-state with tank membership (grouptanks=1)...'); +{ + my ( $total, $filtered, @ids ) = LANraragi::Model::Search::do_search( "", "", -1, 0, 0, 1, 0, 1, 0 ); + my %got = map { $_ => 1 } @ids; + is( $filtered, 2, 'newonly=1 grouptanks should return the new tank and the standalone new archive' ); + ok( $got{"TANK_1589141306"}, 'newonly=1 keeps tank whose member is new' ); + ok( $got{"e4c422fd10943dc169e3489a38cdbf57101a5f7e"}, 'newonly=1 keeps standalone new archive (Rohan)' ); + ok( !$got{"TANK_1589138380"}, 'newonly=1 drops tank with no new member' ); + + ( $total, $filtered, @ids ) = LANraragi::Model::Search::do_search( "", "", -1, 0, 0, -1, 0, 1, 0 ); + %got = map { $_ => 1 } @ids; + is( $filtered, 11, 'newonly=-1 grouptanks should exclude the new tank and the standalone new archive' ); + ok( !$got{"TANK_1589141306"}, 'newonly=-1 drops tank whose member is new' ); + ok( !$got{"e4c422fd10943dc169e3489a38cdbf57101a5f7e"}, 'newonly=-1 drops standalone new archive (Rohan)' ); + ok( $got{"TANK_1589138380"}, 'newonly=-1 keeps tank with no new member' ); +} + done_testing(); diff --git a/tools/Documentation/api-documentation/search-api.md b/tools/Documentation/api-documentation/search-api.md index 867d2d642..38d74334c 100644 --- a/tools/Documentation/api-documentation/search-api.md +++ b/tools/Documentation/api-documentation/search-api.md @@ -18,6 +18,18 @@ description: Perform searches. [OpenAPI lanraragi-api](https://raw.githubusercontent.com/Difegue/LANraragi/refs/heads/dev/tools/openapi.yaml) {% endopenapi-operation %} +{% openapi-operation spec="lanraragi-api" path="/search/composite" method="post" %} +[OpenAPI lanraragi-api](https://raw.githubusercontent.com/Difegue/LANraragi/refs/heads/dev/tools/openapi.yaml) +{% endopenapi-operation %} + +{% openapi-operation spec="lanraragi-api" path="/search/composite/ids" method="post" %} +[OpenAPI lanraragi-api](https://raw.githubusercontent.com/Difegue/LANraragi/refs/heads/dev/tools/openapi.yaml) +{% endopenapi-operation %} + +{% openapi-operation spec="lanraragi-api" path="/search/composite/random" method="post" %} +[OpenAPI lanraragi-api](https://raw.githubusercontent.com/Difegue/LANraragi/refs/heads/dev/tools/openapi.yaml) +{% endopenapi-operation %} + {% openapi-operation spec="lanraragi-api" path="/search/cache" method="delete" %} [OpenAPI lanraragi-api](https://raw.githubusercontent.com/Difegue/LANraragi/refs/heads/dev/tools/openapi.yaml) {% endopenapi-operation %} diff --git a/tools/openapi.yaml b/tools/openapi.yaml index a83ac1e0d..1a37a27a3 100644 --- a/tools/openapi.yaml +++ b/tools/openapi.yaml @@ -3084,6 +3084,233 @@ paths: schema: $ref: '#/components/schemas/OperationResponse' + /search/composite: + post: + security: [] + operationId: compositeSearchArchives + x-mojo-to: api-search#handle_composite + summary: Composite Search Archives + description: >- + Perform a composite search with multiple clauses OR'd together. + Each clause is an independent AND conjunction with its own filter, + categories, and flags. Results are unioned, deduplicated, and + sorted globally. + tags: + - search + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - clauses + properties: + clauses: + type: array + minItems: 1 + maxItems: 10 + description: >- + Array of search clauses. Each clause is an independent AND + conjunction. Multiple clauses are OR-unioned. + items: + $ref: '#/components/schemas/SearchClause' + start: + type: integer + description: >- + Pagination offset. Use -1 to get all results. + sortby: + type: string + description: >- + Namespace by which to sort results. Use "title" for title sort, + "lastread" for last read time sort. + order: + type: string + description: Sort order. + enum: + - asc + - desc + groupby_tanks: + type: boolean + default: true + description: >- + Enable or disable Tankoubon grouping. Defaults to true. + responses: + '200': + description: Search results + content: + application/json: + schema: + type: object + properties: + data: + type: array + description: Archive metadata objects for each search result + items: + $ref: '#/components/schemas/ArchiveMetadataJson' + recordsFiltered: + type: integer + description: Amount of archives in the search result + recordsTotal: + type: integer + description: Total number of archives available. This result will change depending on the `groupby_tanks` parameter. + '204': + description: Search engine is not initialized yet. Please wait a few seconds. + + /search/composite/ids: + post: + security: [] + operationId: compositeSearchArchiveIds + x-mojo-to: api-search#handle_composite_ids + summary: Composite Search Archive IDs + description: >- + Perform a composite search like `/api/search/composite`, but returns + only the ordered list of matching Archive IDs without the accompanying + metadata. + tags: + - search + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - clauses + properties: + clauses: + type: array + minItems: 1 + maxItems: 10 + description: >- + Array of search clauses. Each clause is an independent AND + conjunction. Multiple clauses are OR-unioned. + items: + $ref: '#/components/schemas/SearchClause' + start: + type: integer + description: >- + Pagination offset. Use -1 to get all results. + sortby: + type: string + description: >- + Namespace by which to sort results. Use "title" for title sort, + "lastread" for last read time sort. + order: + type: string + description: Sort order. + enum: + - asc + - desc + groupby_tanks: + type: boolean + default: true + description: >- + Enable or disable Tankoubon grouping. Defaults to true. + responses: + '200': + description: Search results + content: + application/json: + schema: + type: object + properties: + data: + type: array + description: Archive IDs for each search result + items: + type: string + recordsFiltered: + type: integer + description: Amount of archives in the search result + recordsTotal: + type: integer + description: Total number of archives available. This result will change depending on the `groupby_tanks` parameter. + example: + data: + - 28697b96f0ac5858be2614ed10ca47742c9522fd + - 5cf64ec2f3c2c627fb8a64df198af2a8527c4fdb + recordsFiltered: 2 + recordsTotal: 2 + '204': + description: Search engine is not initialized yet. Please wait a few seconds. + + /search/composite/random: + post: + security: [] + operationId: compositeSearchRandomArchives + x-mojo-to: api-search#get_random_archives_composite + summary: Composite Search Random Archives + description: >- + Get randomly selected Archives out of a composite search. Clauses are + OR'd together like `/api/search/composite`, and the requested amount is + sampled at random from the union. + tags: + - search + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - clauses + properties: + clauses: + type: array + minItems: 1 + maxItems: 10 + description: >- + Array of search clauses. Each clause is an independent AND + conjunction. Multiple clauses are OR-unioned. + items: + $ref: '#/components/schemas/SearchClause' + count: + type: integer + default: 5 + description: >- + How many archives you want to pull randomly. Defaults to 5. + + + If the search doesn't return enough data to match your count, you will get the full search shuffled randomly. + groupby_tanks: + type: boolean + default: true + description: >- + Enable or disable Tankoubon grouping. Defaults to true. + responses: + '200': + description: Search results + content: + application/json: + schema: + type: object + properties: + data: + type: array + description: Archive metadata objects for each search result + items: + $ref: '#/components/schemas/ArchiveMetadataJson' + recordsTotal: + type: integer + description: Amount of archives returned. + example: + data: + - arcid: 28697b96f0ac5858be2614ed10ca47742c9522fd + isnew: false + extension: rar + pagecount: 34 + progress: 3 + tags: parody:jojo's bizarre adventure + lastreadtime: 1337038281 + title: Rohan Kishibe goes to Gucci + filename: Gucc + summary: cool summary + size: 0 + recordsTotal: 1 + '204': + description: Search engine is not initialized yet. Please wait a few seconds. + /database/stats: get: security: [] @@ -5371,5 +5598,42 @@ components: properties: url: pattern: '^https://github\.com/[^/]+/[^/]+' + SearchClause: + type: object + description: "Search clause used in composite search." + properties: + filter: + type: string + description: Search filter string (same syntax as /search filter parameter). + categories: + type: array + description: Category filters to apply to this clause. + items: + type: object + required: + - id + - mode + properties: + id: + $ref: '#/components/schemas/CategoryId' + mode: + type: string + description: Whether to include or exclude this category. + enum: + - include + - exclude + newonly: + type: integer + enum: [-1, 0, 1] + description: >- + New archive filter. 1 = only new, -1 = exclude new, 0 = off. + untaggedonly: + type: integer + enum: [-1, 0, 1] + description: >- + Untagged archive filter. 1 = only untagged, -1 = exclude untagged, 0 = off. + hidecompleted: + type: boolean + description: Hide archives where reading progress has reached the end. servers: - url: https://lrr.tvc-16.science/api