Add SYNC_ASSETS_HOOK support

This is useful when having site-local cache shared among several worker
hosts. See added documentation for scenario details.

docs/Installing.asciidoc

@@ -1027,6 +1027,27 @@ comment = openQA test distributions
 systemctl enable --now rsyncd
 --------------------------------------------------------------------------------
 
+=== Alternative caching implementations
+
+Caching described above works well for a single worker host, but in case of
+several hosts in a single site (that is remote from the main openQA webui
+instance) it results in downloading the same assets several times. In
+such case, one can setup local cache on their own (without using
+openqa-worker-cacheservice service) and share it with workers using
+some network filesystem (see <<Installing.asciidoc#Configuring remote workers>>
+section above).
+Such setups can use `SYNC_ASSETS_HOOK` in `/etc/openqa/workers.ini` to ensure
+the cache is up to date before starting the job (or resuming test in developer
+mode). The setting takes a shell command that is executed just before
+evaluating assets. It is up to the system administrator to decide what it
+should do, but there are few suggestions:
+
+* Call rsync, possibly via ssh on the cache host
+* Wait for a lock file signaling that cache download is in progress to disappear
+
+If the command exits with code 32, re-downloading needles in developer mode
+will be skipped.
+
 === Enable linking files referred by job settings
 
 Specific job settings might refer to files within the test distribution.

lib/OpenQA/Worker/Engines/isotovideo.pm

@@ -326,6 +326,12 @@ sub engine_workit ($job, $callback) {
 
     log_debug "Job settings:\n" . format_settings(\%vars);
 
+    # start pre-job hook if any
+    if ($job_settings->{SYNC_ASSETS_HOOK}) {
+        log_debug 'Running SYNC_ASSETS_HOOK';
+        system($job_settings->{SYNC_ASSETS_HOOK});
+    }
+
     # cache/locate assets, set ASSETDIR
     my $assetkeys = detect_asset_keys(\%vars);
     if (my $cache_dir = $global_settings->{CACHEDIRECTORY}) {

t/24-worker-engine.t

@@ -27,6 +27,7 @@ use OpenQA::Test::FakeWorker;
 use Mojo::File qw(path tempdir);
 use Mojo::JSON 'decode_json';
 use OpenQA::Utils qw(testcasedir productdir needledir locate_asset base_host);
+use Cwd qw(getcwd);
 
 # define fake packages for testing asset caching
 {
@@ -511,14 +512,18 @@ subtest 'behavior with ABSOLUTE_TEST_CONFIG_PATHS=1' => sub {
 };
 
 subtest 'link asset' => sub {
+    my $cwd = getcwd;
     my $pool_directory = tempdir('poolXXXX');
     my $worker = OpenQA::Test::FakeWorker->new(pool_directory => $pool_directory);
     my $client = Test::FakeClient->new;
+    # just in case cleanup the symlink to really check if it gets re-created
+    unlink 't/data/openqa/share/factory/hdd/symlink.qcow2' if -e 't/data/openqa/share/factory/hdd/symlink.qcow2';
     my $settings = {
         JOBTOKEN => 'token000',
         ISO => 'openSUSE-13.1-DVD-x86_64-Build0091-Media.iso',
         HDD_1 => 'foo.qcow2',
         HDD_2 => 'symlink.qcow2',
+        SYNC_ASSETS_HOOK => "ln -s foo.qcow2 $cwd/t/data/openqa/share/factory/hdd/symlink.qcow2"
     };
     my $job = OpenQA::Worker::Job->new($worker, $client, {id => 16, settings => $settings});
     combined_like { my $result = _run_engine($job) }
@@ -537,6 +542,7 @@ subtest 'link asset' => sub {
     is $vars_data->{ISO}, 'openSUSE-13.1-DVD-x86_64-Build0091-Media.iso',
       'the value of ISO is basename when doing link';
     is $vars_data->{HDD_1}, 'foo.qcow2', 'the value of HDD_1 is basename when doing link';
+    unlink 't/data/openqa/share/factory/hdd/symlink.qcow2';
 };
 
 subtest 'using cgroupv2' => sub {