Skip to content

Extend FileIOFactory to Improve Customization #724

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Jan 29, 2025
Merged

Extend FileIOFactory to Improve Customization #724

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
ea62b41
Initial Commit
sfc-gh-rxing Jan 3, 2025
decc5b8
Fix test errors
XJDKC Jan 22, 2025
67d7730
Resolved some comments
XJDKC Jan 22, 2025
a5c4557
Resolved some comments
XJDKC Jan 22, 2025
898079e
Removed unused package
XJDKC Jan 22, 2025
9e5cfe0
Make FileIOFactory as an ApplicationScoped bean
XJDKC Jan 23, 2025
2cb4552
Revert some changes
XJDKC Jan 23, 2025
67ab108
Use FileIOFactory to get the creds for cleanup task
XJDKC Jan 23, 2025
c89e831
Use new interface in tests
XJDKC Jan 23, 2025
d8c1804
Merge remote-tracking branch 'apache/main' into pass-internal-propert…
XJDKC Jan 27, 2025
519e0b6
Move some util classes to polaris-service-common as test fixtures
XJDKC Jan 27, 2025
f3a5ef0
Delete the interface in FileIOFactory that is only used for testing
XJDKC Jan 27, 2025
3bb504f
Refactor TestServices and use it in FileIOFactoryTest
XJDKC Jan 28, 2025
ce73197
Small change
XJDKC Jan 28, 2025
5170f6c
Rename TestFileIOFactory to MeasuredFileIOFactory
XJDKC Jan 28, 2025
5a2e046
Use FileIOFactory to load catalog default FileIO in tests, fix log le…
XJDKC Jan 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions quarkus/service/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ dependencies {
testFixturesApi(project(":polaris-tests"))

testImplementation(project(":polaris-api-management-model"))
testImplementation(testFixtures(project(":polaris-service-common")))

testImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests")
testImplementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests")
Expand Down
163 changes: 0 additions & 163 deletions quarkus/service/src/test/java/org/apache/polaris/service/quarkus/TestServices.java
View file
Open in desktop

This file was deleted.

9 changes: 4 additions & 5 deletions ...service/src/test/java/org/apache/polaris/service/quarkus/admin/ManagementServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,14 @@
import org.apache.polaris.core.admin.model.PolarisCatalog;
import org.apache.polaris.core.admin.model.StorageConfigInfo;
import org.apache.polaris.core.admin.model.UpdateCatalogRequest;
import org.apache.polaris.service.quarkus.TestServices;
import org.apache.polaris.service.quarkus.catalog.io.TestFileIOFactory;
import org.apache.polaris.service.TestServices;
import org.junit.jupiter.api.Test;

public class ManagementServiceTest {
static TestServices services =
TestServices.inMemory(
new TestFileIOFactory(),
Map.of("SUPPORTED_CATALOG_STORAGE_TYPES", List.of("S3", "GCS", "AZURE")));
TestServices.builder()
.config(Map.of("SUPPORTED_CATALOG_STORAGE_TYPES", List.of("S3", "GCS", "AZURE")))
.build();

@Test
public void testCreateCatalogWithDisallowedStorageConfig() {
Expand Down
8 changes: 6 additions & 2 deletions .../service/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisAuthzTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,11 @@
import org.apache.polaris.core.persistence.PolarisMetaStoreSession;
import org.apache.polaris.service.admin.PolarisAdminService;
import org.apache.polaris.service.catalog.BasePolarisCatalog;
import org.apache.polaris.service.catalog.PolarisPassthroughResolutionView;
import org.apache.polaris.service.catalog.io.DefaultFileIOFactory;
import org.apache.polaris.service.catalog.io.FileIOFactory;
import org.apache.polaris.service.config.DefaultConfigurationStore;
import org.apache.polaris.service.config.RealmEntityManagerFactory;
import org.apache.polaris.service.quarkus.catalog.PolarisPassthroughResolutionView;
import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.AfterEach;
Expand Down Expand Up @@ -160,6 +161,7 @@ public Map<String, String> getConfigOverrides() {
protected PolarisEntityManager entityManager;
protected PolarisMetaStoreManager metaStoreManager;
protected PolarisMetaStoreSession metaStoreSession;
protected FileIOFactory fileIOFactory;
protected PolarisBaseEntity catalogEntity;
protected PrincipalEntity principalEntity;
protected RealmId realmId;
Expand Down Expand Up @@ -385,6 +387,8 @@ private void initBaseCatalog() {
PolarisPassthroughResolutionView passthroughView =
new PolarisPassthroughResolutionView(
entityManager, metaStoreSession, securityContext, CATALOG_NAME);
this.fileIOFactory =
new DefaultFileIOFactory(realmEntityManagerFactory, managerFactory, configurationStore);
this.baseCatalog =
new BasePolarisCatalog(
realmId,
Expand All @@ -396,7 +400,7 @@ private void initBaseCatalog() {
passthroughView,
securityContext,
Mockito.mock(),
new DefaultFileIOFactory());
fileIOFactory);
this.baseCatalog.initialize(
CATALOG_NAME,
ImmutableMap.of(
Expand Down
6 changes: 2 additions & 4 deletions ...src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingCatalogTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,16 +33,14 @@
import org.apache.polaris.core.admin.model.CatalogProperties;
import org.apache.polaris.core.admin.model.CreateCatalogRequest;
import org.apache.polaris.core.admin.model.StorageConfigInfo;
import org.apache.polaris.service.quarkus.TestServices;
import org.apache.polaris.service.quarkus.catalog.io.TestFileIOFactory;
import org.apache.polaris.service.TestServices;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;

public class PolarisOverlappingCatalogTest {

static TestServices services =
TestServices.inMemory(
new TestFileIOFactory(), Map.of("ALLOW_OVERLAPPING_CATALOG_URLS", "false"));
TestServices.builder().config(Map.of("ALLOW_OVERLAPPING_CATALOG_URLS", "false")).build();

private Response createCatalog(String prefix, String defaultBaseLocation, boolean isExternal) {
return createCatalog(prefix, defaultBaseLocation, isExternal, new ArrayList<String>());
Expand Down
4 changes: 2 additions & 2 deletions ...e/src/test/java/org/apache/polaris/service/quarkus/admin/PolarisOverlappingTableTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import org.apache.polaris.core.admin.model.CreateCatalogRequest;
import org.apache.polaris.core.admin.model.FileStorageConfigInfo;
import org.apache.polaris.core.admin.model.StorageConfigInfo;
import org.apache.polaris.service.quarkus.TestServices;
import org.apache.polaris.service.TestServices;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
Expand Down Expand Up @@ -105,7 +105,7 @@ void testTableLocationRestrictions(
Map<String, Object> serverConfig,
Map<String, String> catalogConfig,
int expectedStatusForOverlaps) {
TestServices services = TestServices.inMemory(serverConfig);
TestServices services = TestServices.builder().config(serverConfig).build();

CatalogProperties.Builder propertiesBuilder =
CatalogProperties.builder()
Expand Down
Loading