Skip to content

Commit

Permalink
(#565) Encode/Decode computed tag rules when saving/loading them to t…
Browse files Browse the repository at this point in the history 
…he database
  • Loading branch information
@Difegue
Difegue committed Dec 22, 2021
1 parent b511aed commit 2465b56
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 115 deletions.
11 changes: 9 additions & 2 deletions lib/LANraragi/Utils/Database.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,13 @@ sub save_computed_tagrules {
my ($tagrules) = @_;
my $redis = LANraragi::Model::Config->get_redis;
$redis->del("LRR_TAGRULES");
$redis->lpush( "LRR_TAGRULES", reverse flat(@$tagrules) ) if (@$tagrules);

if (@$tagrules) {
my @flat = reverse flat(@$tagrules);
my @encoded_flat = map { redis_encode($_) } @flat;
$redis->lpush( "LRR_TAGRULES", @encoded_flat );
}

$redis->quit();
return;
}
Expand All @@ -385,7 +391,8 @@ sub get_computed_tagrules {

if ( $redis->exists("LRR_TAGRULES") ) {
my @flattened_rules = $redis->lrange( "LRR_TAGRULES", 0, -1 );
@tagrules = unflat_tagrules( \@flattened_rules );
my @decoded_rules = map { redis_decode($_) } @flattened_rules;
@tagrules = unflat_tagrules( \@decoded_rules );
} else {
@tagrules = tags_rules_to_array( restore_CRLF( LANraragi::Model::Config->get_tagrules ) );
$redis->lpush( "LRR_TAGRULES", reverse flat(@tagrules) ) if (@tagrules);
Expand Down
180 changes: 67 additions & 113 deletions tests/LANraragi/Utils/Tags.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,21 +13,14 @@ use Test::Deep;
my $cwd = getcwd();

my @incoming_tags = (
'group:alpha',
'group:beta',
'namespace:ONE',
'namespace:Two and Three',
'namespace-fake',
'flip',
'ping',
'cat',
'with space',
'SCREAM'
'group:alpha', 'group:beta', 'namespace:ONE', 'namespace:Two and Three',
'namespace-fake', 'flip', 'ping', 'cat',
'with space', 'SCREAM'
);

BEGIN { use_ok('LANraragi::Utils::Tags'); }

note ("testing tag rules conversion...");
note("testing tag rules conversion...");

{
my $text_rules = ' ping -> pong
Expand All @@ -37,23 +30,35 @@ note ("testing tag rules conversion...");

my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

cmp_deeply( \@rules, [ [ 'replace', 'ping', 'pong' ], [ 'replace', 'no-space', 'keep spaces' ] ], 'skips empty lines and removes surrounding spaces');
cmp_deeply(
\@rules,
[ [ 'replace', 'ping', 'pong' ], [ 'replace', 'no-space', 'keep spaces' ] ],
'skips empty lines and removes surrounding spaces'
);
}

{
my $text_rules = 'SCREAM -> Be Quite';

my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

cmp_deeply( \@rules, [ [ 'replace', 'scream', 'Be Quite' ] ], 'always converts the match term to lowercase');
cmp_deeply( \@rules, [ [ 'replace', 'scream', 'Be Quite' ] ], 'always converts the match term to lowercase' );
}

{
my $text_rules = "\r\n ping -> pong\r\n -cat";

my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

cmp_deeply( \@rules, [ [ 'replace', 'ping', 'pong' ], [ 'remove', 'cat', '' ] ], 'handles CRLF');
cmp_deeply( \@rules, [ [ 'replace', 'ping', 'pong' ], [ 'remove', 'cat', '' ] ], 'handles CRLF' );
}

{
my $text_rules = 'ping -> おはよう';

my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

cmp_deeply( \@rules, [ [ 'replace', 'ping', 'おはよう' ] ], 'handles non-ASCII characters' );
}

{
Expand All @@ -62,176 +67,138 @@ note ("testing tag rules conversion...");

my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

cmp_deeply( \@rules, [ [ 'remove', 'ping', '' ], [ 'remove', 'flip', '' ] ], 'blacklist mode');
cmp_deeply( \@rules, [ [ 'remove', 'ping', '' ], [ 'remove', 'flip', '' ] ], 'blacklist mode' );
}

{
my $text_rules = 'ping -> pong';

my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

cmp_deeply( \@rules, [ [ 'replace', 'ping', 'pong' ] ], 'simple substitution');
cmp_deeply( \@rules, [ [ 'replace', 'ping', 'pong' ] ], 'simple substitution' );
}

{
my $text_rules = 'group:alpha -> lone wolf';

my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

cmp_deeply( \@rules, [ [ 'replace', 'group:alpha', 'lone wolf' ] ], 'simple substitution with namespace');
cmp_deeply( \@rules, [ [ 'replace', 'group:alpha', 'lone wolf' ] ], 'simple substitution with namespace' );
}

{
my $text_rules = 'group:* -> block:*';

my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

cmp_deeply( \@rules, [ [ 'replace_ns', 'group', 'block' ] ], 'namespace substitution');
cmp_deeply( \@rules, [ [ 'replace_ns', 'group', 'block' ] ], 'namespace substitution' );
}

{
my $text_rules = '-with space';

my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

cmp_deeply( \@rules, [ [ 'remove', 'with space', '' ] ], 'simple deletion');
cmp_deeply( \@rules, [ [ 'remove', 'with space', '' ] ], 'simple deletion' );
}

{
my $text_rules = '-namespace:*';

my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

cmp_deeply( \@rules, [ [ 'remove_ns', 'namespace', '' ] ], 'namespace deletion');
cmp_deeply( \@rules, [ [ 'remove_ns', 'namespace', '' ] ], 'namespace deletion' );
}

{
my $text_rules = '~namespace';

my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

cmp_deeply( \@rules, [ [ 'strip_ns', 'namespace', '' ] ], 'strips namespace');
cmp_deeply( \@rules, [ [ 'strip_ns', 'namespace', '' ] ], 'strips namespace' );
}

note("testing tags manipulation ...");

{
my @tags = LANraragi::Utils::Tags::rewrite_tags(\@incoming_tags, []);
my @tags = LANraragi::Utils::Tags::rewrite_tags( \@incoming_tags, [] );

cmp_deeply( \@tags, \@incoming_tags, 'no rule');
cmp_deeply( \@tags, \@incoming_tags, 'no rule' );
}

{
my $text_rules = 'flip -> flop
cat -> dog';
my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

my @tags = LANraragi::Utils::Tags::rewrite_tags(\@incoming_tags, \@rules);
my @tags = LANraragi::Utils::Tags::rewrite_tags( \@incoming_tags, \@rules );

cmp_deeply(
\@tags,
[
'group:alpha',
'group:beta',
'namespace:ONE',
'namespace:Two and Three',
'namespace-fake',
'flop',
'ping',
'dog',
'with space',
'SCREAM'
[ 'group:alpha', 'group:beta', 'namespace:ONE', 'namespace:Two and Three',
'namespace-fake', 'flop', 'ping', 'dog',
'with space', 'SCREAM'
],
'simple substitution');
'simple substitution'
);
}

{
my $text_rules = '-cat';
my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);
my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

my @tags = LANraragi::Utils::Tags::rewrite_tags(\@incoming_tags, \@rules);
my @tags = LANraragi::Utils::Tags::rewrite_tags( \@incoming_tags, \@rules );

cmp_deeply(
\@tags,
[
'group:alpha',
'group:beta',
'namespace:ONE',
'namespace:Two and Three',
'namespace-fake',
'flip',
'ping',
'with space',
[ 'group:alpha', 'group:beta', 'namespace:ONE', 'namespace:Two and Three',
'namespace-fake', 'flip', 'ping', 'with space',
'SCREAM'
],
'simple deletion');
'simple deletion'
);
}

{
my $text_rules = '-namespace:*';
my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);
my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

my @tags = LANraragi::Utils::Tags::rewrite_tags(\@incoming_tags, \@rules);
my @tags = LANraragi::Utils::Tags::rewrite_tags( \@incoming_tags, \@rules );

cmp_deeply(
\@tags,
[
'group:alpha',
'group:beta',
'namespace-fake',
'flip',
'ping',
'cat',
'with space',
'SCREAM'
],
'namespace deletion');
[ 'group:alpha', 'group:beta', 'namespace-fake', 'flip', 'ping', 'cat', 'with space', 'SCREAM' ],
'namespace deletion'
);
}

{
my $text_rules = 'group:* -> block:*';
my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);
my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

my @tags = LANraragi::Utils::Tags::rewrite_tags(\@incoming_tags, \@rules);
my @tags = LANraragi::Utils::Tags::rewrite_tags( \@incoming_tags, \@rules );

cmp_deeply(
\@tags,
[
'block:alpha',
'block:beta',
'namespace:ONE',
'namespace:Two and Three',
'namespace-fake',
'flip',
'ping',
'cat',
'with space',
'SCREAM'
[ 'block:alpha', 'block:beta', 'namespace:ONE', 'namespace:Two and Three',
'namespace-fake', 'flip', 'ping', 'cat',
'with space', 'SCREAM'
],
'namespace substitution');
'namespace substitution'
);
}

{
my $text_rules = '~namespace';
my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);
my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

my @tags = LANraragi::Utils::Tags::rewrite_tags(\@incoming_tags, \@rules);
my @tags = LANraragi::Utils::Tags::rewrite_tags( \@incoming_tags, \@rules );

cmp_deeply(
\@tags,
[
'group:alpha',
'group:beta',
'ONE',
'Two and Three',
'namespace-fake',
'flip',
'ping',
'cat',
'with space',
'SCREAM'
],
'strips namespace');
[ 'group:alpha', 'group:beta', 'ONE', 'Two and Three', 'namespace-fake', 'flip', 'ping', 'cat', 'with space', 'SCREAM' ],
'strips namespace'
);
}

{
Expand All @@ -240,45 +207,32 @@ note("testing tags manipulation ...");
-PING';
my @rules = LANraragi::Utils::Tags::tags_rules_to_array($text_rules);

my @tags = LANraragi::Utils::Tags::rewrite_tags(\@incoming_tags, \@rules);
my @tags = LANraragi::Utils::Tags::rewrite_tags( \@incoming_tags, \@rules );

cmp_deeply(
\@tags,
[
'group:alpha',
'group:beta',
'ONE',
'Two and Three',
'namespace-fake',
'flip',
'cat',
'with space',
'Please Stop'
],
'rules matching is case insensitive');
[ 'group:alpha', 'group:beta', 'ONE', 'Two and Three', 'namespace-fake', 'flip', 'cat', 'with space', 'Please Stop' ],
'rules matching is case insensitive'
);
}

note('testing rules unflattening...');

{
my $expected_rules = [
[ 'strip_ns', 'namespace', '' ],
[ 'replace', 'scream', 'Please Stop' ],
[ 'remove', 'ping', '']
];
my $expected_rules = [ [ 'strip_ns', 'namespace', '' ], [ 'replace', 'scream', 'Please Stop' ], [ 'remove', 'ping', '' ] ];
my @flattened_rules = flat(@$expected_rules);

my @rules = LANraragi::Utils::Tags::unflat_tagrules(\@flattened_rules);
my @rules = LANraragi::Utils::Tags::unflat_tagrules( \@flattened_rules );

cmp_deeply( \@rules, $expected_rules, 'unflattened rules');
cmp_deeply( \@rules, $expected_rules, 'unflattened rules' );
}

{
my @empty_rules;
my @rules = LANraragi::Utils::Tags::unflat_tagrules(\@empty_rules);
cmp_deeply( \@rules, [ ], 'unflattened empty rules');
my @rules = LANraragi::Utils::Tags::unflat_tagrules( \@empty_rules );
cmp_deeply( \@rules, [], 'unflattened empty rules' );
my @rules = LANraragi::Utils::Tags::unflat_tagrules(undef);
cmp_deeply( \@rules, [ ], 'unflattened undef array');
cmp_deeply( \@rules, [], 'unflattened undef array' );
}

done_testing();

0 comments on commit 2465b56

Please sign in to comment.