Bootstrap test data from JSON fixtures instead of SQL migrations

fixtures/0100_user-accounts.json

@@ -4,7 +4,7 @@
     "uuid":  "7e64818d-6276-46fb-8bb1-732e6e09f7e9",
     "firstName": "Albert",
     "lastName": "Einstein",
-    "email": "albert.einstein@example.org",
+    "email": "albert.einstein@example.com",
     "passwordHash": "$2a$10$hZF1abbZ48Tf.3RndC9W6OlDt6gnBoD/2HbzJayTs6be7d.5DbpnW",
     "role": "ADMIN"
   },
@@ -13,7 +13,7 @@
     "uuid":  "b5b92c69-5ed9-4054-954d-0121c29b6800",
     "firstName": "Nikola",
     "lastName": "Tesla",
-    "email": "nikola.tesla@example.org",
+    "email": "nikola.tesla@example.com",
     "passwordHash": "$2a$10$tMbZUZg9AbYL514R.hZ0tuzvfZJR5NQhSVeJPTQhNwPf6gv/cvrna",
     "role": "USER"
   }

src/main/java/org/fairdatapoint/config/BootstrapConfig.java

@@ -38,7 +38,6 @@
 import org.springframework.stereotype.Component;
 
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
@@ -75,12 +74,15 @@ public Jackson2RepositoryPopulatorFactoryBean repositoryPopulator() {
             log.info("Bootstrap repository populator enabled");
             try {
                 // collect fixture resources
-                log.info("Looking for db fixtures in the following directories: {}",
-                        String.join(", ", this.bootstrap.getDbFixturesDirs()));
-                for (String fixturesDir : this.bootstrap.getDbFixturesDirs()) {
-                    // Path.of() removes trailing slashes, so it is safe to concatenate "/*.json".
-                    // Note that Path.of(fixturesDir).resolve("*.json") could work on unix but fails on windows.
-                    final String locationPattern = "file:" + Path.of(fixturesDir) + "/*.json";
+                log.info("Looking for db fixtures in the following locations: {}",
+                        String.join(", ", this.bootstrap.getLocations()));
+                for (String location : this.bootstrap.getLocations()) {
+                    // Only look for JSON files
+                    String locationPattern = location;
+                    if (!locationPattern.endsWith(".json")) {
+                        // naive append may lead to redundant slashes, but the OS ignores those
+                        locationPattern += "/*.json";
+                    }
                     resources.addAll(List.of(resourceResolver.getResources(locationPattern)));
                 }
                 // remove resources that have been applied already
@@ -111,7 +113,7 @@ public Jackson2RepositoryPopulatorFactoryBean repositoryPopulator() {
     public class RepositoriesPopulatedEventListener implements ApplicationListener<RepositoriesPopulatedEvent> {
         @Override
         public void onApplicationEvent(@NotNull RepositoriesPopulatedEvent event) {
-            log.info("Repositories populated");
+            log.info("Repository populator finished.");
             // Create fixture history records for all resources that have been applied.
             // Note: This assumes that all items in the resources list have been *successfully* applied. However, I'm
             // not sure if this can be guaranteed. If it does turn out to be a problem, we could try e.g. extending the

src/main/java/org/fairdatapoint/config/properties/BootstrapProperties.java

@@ -38,6 +38,7 @@
 public class BootstrapProperties {
     // boolean defaults to false
     private boolean enabled;
-    // directories relative to project root
-    private List<String> dbFixturesDirs;
+    // locations to search for fixtures, for example, file:fixtures, relative to project root,
+    // or classpath:fixtures (see PathMatchingResourcePatternResolver docs for valid patterns)
+    private List<String> locations;
 }

src/main/java/org/fairdatapoint/database/rdf/migration/development/metadata/RdfMetadataMigration.java

@@ -22,6 +22,7 @@
  */
 package org.fairdatapoint.database.rdf.migration.development.metadata;
 
+import lombok.extern.slf4j.Slf4j;
 import org.fairdatapoint.api.dto.metadata.MetaStateChangeDTO;
 import org.fairdatapoint.database.common.migration.Migration;
 import org.fairdatapoint.database.db.repository.ResourceDefinitionRepository;
@@ -49,6 +50,7 @@
 import static org.fairdatapoint.entity.metadata.MetadataGetter.getUri;
 import static org.fairdatapoint.util.ValueFactoryHelper.i;
 
+@Slf4j
 @Service
 public class RdfMetadataMigration implements Migration {
 
@@ -104,8 +106,8 @@ public void runMigration() {
             // Load metadata fixtures
             importDefaultFixtures(persistentUrl);
         }
-        catch (MetadataServiceException exception) {
-            exception.printStackTrace();
+        catch (Exception exception) {
+            log.warn("Failed to run RDF development migration:", exception);
         }
     }
 

src/main/resources/application.yml

@@ -114,5 +114,5 @@ server:
 
 bootstrap:
   enabled: true
-  db-fixtures-dirs:
-    - "fixtures"
+  locations:
+    - file:fixtures

src/test/java/org/fairdatapoint/WebIntegrationTest.java

@@ -30,6 +30,9 @@
 import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.test.web.client.TestRestTemplate;
+import org.springframework.context.ApplicationContext;
+import org.springframework.data.repository.init.ResourceReaderRepositoryPopulator;
+import org.springframework.data.repository.support.Repositories;
 import org.springframework.test.annotation.DirtiesContext;
 import org.springframework.test.context.ActiveProfiles;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
@@ -64,10 +67,22 @@ public abstract class WebIntegrationTest {
     @Autowired
     protected AclMigration aclMigration;
 
+    @Autowired
+    private ApplicationContext applicationContext;
+
+    @Autowired
+    private ResourceReaderRepositoryPopulator populator;
+
     @BeforeEach
     public void setup() {
+        // drop test database content
         flyway.clean();
+        // re-migrate schemas
         flyway.migrate();
+        // re-populate the database using fixtures
+        populator.populate(new Repositories(applicationContext));
+        // re-migrate acl data
+        // (TODO: AclMigration is in a subfolder of rdf/migration, but is it even related to rdf? Looks relational...)
         aclMigration.runMigration();
     }
 }

src/test/java/org/fairdatapoint/acceptance/search/query/saved/Detail_DELETE.java

@@ -70,7 +70,7 @@ public void res403_anonymousUser() {
 
         // THEN:
         assertThat(result.getStatusCode(), is(equalTo(HttpStatus.FORBIDDEN)));
-        assertThat(searchSavedQueryRepository.count(), is(equalTo(3L)));
+        assertThat(searchSavedQueryRepository.count(), is(equalTo(4L)));
     }
 
     @Test
@@ -92,7 +92,7 @@ public void res403_nonOwnerUser() {
 
         // THEN:
         assertThat(result.getStatusCode(), is(equalTo(HttpStatus.FORBIDDEN)));
-        assertThat(searchSavedQueryRepository.count(), is(equalTo(3L)));
+        assertThat(searchSavedQueryRepository.count(), is(equalTo(4L)));
     }
 
     @Test
@@ -114,7 +114,7 @@ public void res200_owner() {
 
         // THEN:
         assertThat(result.getStatusCode(), is(equalTo(HttpStatus.NO_CONTENT)));
-        assertThat(searchSavedQueryRepository.count(), is(equalTo(2L)));
+        assertThat(searchSavedQueryRepository.count(), is(equalTo(3L)));
     }
 
     @Test
@@ -136,6 +136,6 @@ public void res200_admin() {
 
         // THEN:
         assertThat(result.getStatusCode(), is(equalTo(HttpStatus.NO_CONTENT)));
-        assertThat(searchSavedQueryRepository.count(), is(equalTo(2L)));
+        assertThat(searchSavedQueryRepository.count(), is(equalTo(3L)));
     }
 }

src/test/java/org/fairdatapoint/acceptance/search/query/saved/List_GET.java

@@ -155,7 +155,7 @@ public void res200_admin() {
         // THEN:
         assertThat(result.getStatusCode(), is(equalTo(HttpStatus.OK)));
         List<SearchSavedQueryDTO> body = result.getBody();
-        assertThat(body.size(), is(equalTo(3)));
+        assertThat(body.size(), is(equalTo(4)));
         assertThat(body.get(0).getUuid(), is(equalTo(q1.getUuid())));
         assertThat(body.get(1).getUuid(), is(equalTo(q2.getUuid())));
         assertThat(body.get(2).getUuid(), is(equalTo(q3.getUuid())));

src/test/java/org/fairdatapoint/database/db/repository/bootstrap/DatabaseBootstrapTests.java

@@ -31,24 +31,19 @@
 import org.fairdatapoint.entity.apikey.ApiKey;
 import org.fairdatapoint.entity.search.SearchSavedQuery;
 import org.fairdatapoint.entity.user.UserAccount;
+import org.fairdatapoint.util.KnownUUIDs;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.autoconfigure.orm.jpa.AutoConfigureTestEntityManager;
-import org.springframework.test.context.TestPropertySource;
 
 import java.util.Optional;
 import java.util.UUID;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 @AutoConfigureTestEntityManager
 @Transactional
-@TestPropertySource(
-        properties = """
-            bootstrap.enabled=true
-            bootstrap.db-fixtures-dirs=src/test/resources/fixtures
-        """
-)
 public class DatabaseBootstrapTests extends BaseIntegrationTest {
     @Autowired
     private UserAccountRepository userAccountRepository;
@@ -59,28 +54,33 @@ public class DatabaseBootstrapTests extends BaseIntegrationTest {
     @Autowired
     private SearchSavedQueryRepository searchSavedQueryRepository;
 
+    private final String einsteinEmail = "[email protected]";
+
     @Test
     public void testSingleEntityBootstrap() {
-        final Optional<UserAccount> userAccount = userAccountRepository.findByEmail("[email protected]");
-        assertEquals(true, userAccount.isPresent());
-        assertEquals("John", userAccount.get().getFirstName());
-        assertEquals("Doe", userAccount.get().getLastName());
-        assertEquals(UUID.fromString("e8f98d8e-0c4f-4a4b-9cc7-dd884f0c75ee"), userAccount.get().getUuid());
+        final Optional<UserAccount> userAccount = userAccountRepository.findByEmail(einsteinEmail);
+        assertTrue(userAccount.isPresent());
+        assertEquals("Albert", userAccount.get().getFirstName());
+        assertEquals("Einstein", userAccount.get().getLastName());
+        assertEquals(KnownUUIDs.USER_ALBERT_UUID, userAccount.get().getUuid());
     }
 
     @Test
     public void testRelatedEntityBootstrap() {
-        final Optional<ApiKey> apiKey = apiKeyRepository.findByToken("testing-token");
-        assertEquals(true, apiKey.isPresent());
-        assertEquals("[email protected]", apiKey.get().getUserAccount().getEmail());
-        assertEquals(UUID.fromString("9d734008-91bb-47e3-97aa-2f537e67d9e6"), apiKey.get().getUuid());
+        final UUID einsteinApiKeyUuid = UUID.fromString("a1c00673-24c5-4e0a-bdbe-22e961ee7548");
+        final String einsteinApiKeyToken = "a274793046e34a219fd0ea6362fcca61a001500b71724f4c973a017031653c20";
+        final Optional<ApiKey> apiKey = apiKeyRepository.findByToken(einsteinApiKeyToken);
+        assertTrue(apiKey.isPresent());
+        assertEquals(einsteinEmail, apiKey.get().getUserAccount().getEmail());
+        assertEquals(einsteinApiKeyUuid, apiKey.get().getUuid());
     }
 
     @Test
     public void testDuplicateIdEntityOverwriteBootstrap() {
-        final Optional<SearchSavedQuery> savedQuery = searchSavedQueryRepository.findByUuid(UUID.fromString("4c57eff3-4608-40ae-85af-b442cfea0746"));
-        assertEquals(true, savedQuery.isPresent());
-        assertEquals("[email protected]", savedQuery.get().getUserAccount().getEmail());
+        final Optional<SearchSavedQuery> savedQuery = searchSavedQueryRepository.findByUuid(
+                UUID.fromString("4c57eff3-4608-40ae-85af-b442cfea0746"));
+        assertTrue(savedQuery.isPresent());
+        assertEquals("[email protected]", savedQuery.get().getUserAccount().getEmail());
         assertEquals("Some query 2", savedQuery.get().getName());
     }
 }

src/test/java/org/fairdatapoint/service/resource/ResourceDefinitionCacheTest.java

@@ -32,6 +32,9 @@
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.ApplicationContext;
+import org.springframework.data.repository.init.ResourceReaderRepositoryPopulator;
+import org.springframework.data.repository.support.Repositories;
 import org.springframework.test.context.ActiveProfiles;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -53,10 +56,18 @@ public class ResourceDefinitionCacheTest extends BaseIntegrationTest {
     @Autowired
     protected Flyway flyway;
 
+    @Autowired
+    private ApplicationContext applicationContext;
+
+    @Autowired
+    private ResourceReaderRepositoryPopulator populator;
+
     @BeforeEach
     public void setup() {
         flyway.clean();
         flyway.migrate();
+        // re-populate the database using default fixtures
+        populator.populate(new Repositories(applicationContext));
     }
 
     @Test

src/test/resources/application-testing.yml

@@ -17,9 +17,15 @@ spring:
     username: ${FDP_POSTGRES_USERNAME:fdp}
     password: ${FDP_POSTGRES_PASSWORD:fdp}
   flyway:
-    locations: classpath:test/db/migration,classpath:db/migration
+    locations: classpath:db/migration
     fail-on-missing-locations: true
     clean-disabled: false
 
 ping:
   enabled: false
+
+bootstrap:
+  enabled: true
+  locations:
+    - file:fixtures
+    - classpath:test-fixtures

src/test/resources/log4j2.xml → src/test/resources/log4j2-test.xml

@@ -6,8 +6,11 @@
         </Console>
     </Appenders>
     <Loggers>
-        <!-- <Logger name="org.fairdatapoint.api.controller.exception" level="DEBUG"/> -->
+        <!-- set log levels for individual packages (add or change as needed) -->
         <Logger name="org.springframework" level="off"/>
-        <Root level="off"/>
+        <!-- set root log level to limit the amount of test output clutter -->
+        <Root level="off">
+            <AppenderRef ref="console-log"/>
+        </Root>
     </Loggers>
 </Configuration>