Skip to content

CSCTTV-4517 publication abstract db change #245

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sarkikos merged 3 commits into from
Sep 30, 2025
Merged

CSCTTV-4517 publication abstract db change #245

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c25a87a
CSCTTV-4517 DimPublication model change. Abstract is take from DimDes…
sarkikos Sep 29, 2025
9312760
Reduce DB query batch size in indexing from 30000 to 3000
sarkikos Sep 29, 2025
e4adbca
Increase DB query batch size in indexing from 3000 to 15000
sarkikos Sep 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion aspnetcore/src/DatabaseContext/ApiDbContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1294,6 +1294,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)

entity.ToTable("dim_identifierless_data");

entity.HasIndex(e => e.DimIdentifierlessDataId, "idx_dim_identifierless_data_id");

entity.Property(e => e.Id).HasColumnName("id");
entity.Property(e => e.Created)
.HasColumnType("datetime")
Expand Down Expand Up @@ -1410,6 +1412,12 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)

entity.ToTable("dim_keyword");

entity.HasIndex(e => e.DimKeywordCloseMatch, "idx_dim_keyword_close_match");

entity.HasIndex(e => e.DimKeywordLanguageVariant, "idx_dim_keyword_language_variant");

entity.HasIndex(e => e.DimKeywordRelated, "idx_dim_keyword_related");

entity.Property(e => e.Id).HasColumnName("id");
entity.Property(e => e.ConceptUri)
.HasMaxLength(255)
Expand Down Expand Up @@ -1981,6 +1989,12 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)

entity.ToTable("dim_pid");

entity.HasIndex(e => e.DimProfileOnlyDatasetId, "idx_dim_profile_only_dataset_id");

entity.HasIndex(e => e.DimProfileOnlyFundingDecisionId, "idx_dim_profile_only_funding_decision_id");

entity.HasIndex(e => e.DimProfileOnlyPublicationId, "idx_dim_profile_only_publication_id");

entity.Property(e => e.Id).HasColumnName("id");
entity.Property(e => e.Created)
.HasColumnType("datetime")
Expand Down Expand Up @@ -2478,7 +2492,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
entity.ToTable("dim_publication");

entity.Property(e => e.Id).HasColumnName("id");
entity.Property(e => e.Abstract).HasColumnName("abstract");
entity.Property(e => e.ApcFeeEur)
.HasColumnType("decimal(18, 2)")
.HasColumnName("apc_fee_EUR");
Expand Down Expand Up @@ -3120,6 +3133,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
{
j.HasKey("DimResearchDatasetId", "DimKeywordId").HasName("PK__br_resea__4D226DF29D73B9C7");
j.ToTable("br_research_dataset_dim_keyword");
j.HasIndex(new[] { "DimKeywordId" }, "idx_dim_keyword_id");
j.IndexerProperty<int>("DimResearchDatasetId").HasColumnName("dim_research_dataset_id");
j.IndexerProperty<int>("DimKeywordId").HasColumnName("dim_keyword_id");
});
Expand Down Expand Up @@ -3591,6 +3605,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
.HasColumnName("created");
entity.Property(e => e.DimKnownPersonId).HasColumnName("dim_known_person_id");
entity.Property(e => e.Hidden).HasColumnName("hidden");
entity.Property(e => e.HighlightOpeness).HasColumnName("highlight_openess");
entity.Property(e => e.Modified)
.HasColumnType("datetime")
.HasColumnName("modified");
Expand Down Expand Up @@ -3747,6 +3762,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)

entity.ToTable("fact_contribution");

entity.HasIndex(e => e.DimNameId, "fact_contribution_dim_name_id");

entity.Property(e => e.DimFundingDecisionId).HasColumnName("dim_funding_decision_id");
entity.Property(e => e.DimOrganizationId).HasColumnName("dim_organization_id");
entity.Property(e => e.DimDateId).HasColumnName("dim_date_id");
Expand Down Expand Up @@ -4212,6 +4229,8 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)

entity.ToTable("fact_relation");

entity.HasIndex(e => e.FromIdentifierlessDataId, "idx_from_identifierless_data_id");

entity.Property(e => e.RelationTypeCode).HasColumnName("relation_type_code");
entity.Property(e => e.FromPublicationId).HasColumnName("from_publication_id");
entity.Property(e => e.FromResearchDatasetId).HasColumnName("from_research_dataset_id");
Expand Down
2 changes: 0 additions & 2 deletions aspnetcore/src/DatabaseContext/Entities/DimPublication.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,8 +93,6 @@ public partial class DimPublication

public int PublisherOpenAccessCode { get; set; }

public string? Abstract { get; set; }

public int DimPublicationChannelId { get; set; }

public int JufoClass { get; set; }
Expand Down
2 changes: 2 additions & 0 deletions aspnetcore/src/DatabaseContext/Entities/DimUserProfile.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ public partial class DimUserProfile

public bool PublishNewOrcidData { get; set; }

public bool HighlightOpeness { get; set; }

public virtual ICollection<BrGrantedPermission> BrGrantedPermissions { get; set; } = new List<BrGrantedPermission>();

public virtual ICollection<DimFieldDisplaySetting> DimFieldDisplaySettings { get; set; } = new List<DimFieldDisplaySetting>();
Expand Down
2 changes: 1 addition & 1 deletion aspnetcore/src/Indexer/Indexer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ private async Task IndexEntities(string indexName, IIndexRepository repository,
*/

int skipAmount = 0;
int takeAmount = 30000;
int takeAmount = 15000;
int numOfResults = 0;
int processedCount = 0;

Expand Down
39 changes: 21 additions & 18 deletions aspnetcore/src/Repositories/Maps/PublicationProfile.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,22 @@ public class PublicationProfile : Profile
{
private const string PreprintType = "preprint";
private const string SelfArchivedType = "self_archived";
private const string DescriptiveItemType_Abstract = "abstract";

public PublicationProfile()
{
AllowNullCollections = true;
AllowNullDestinationValues = true;

CreateProjection<DimPublication, Publication>()
.AddTransform<string?>(s => string.IsNullOrWhiteSpace(s) ? null : s)
.ForMember(dst => dst.ExportSortId, opt => opt.MapFrom(src => (long)src.Id))
.ForMember(dst => dst.Id, opt => opt.MapFrom(src => src.PublicationId))
.ForMember(dst => dst.OriginalPublicationId, opt => opt.MapFrom(src => src.OriginalPublicationId))
.ForMember(dst => dst.Name, opt => opt.MapFrom(src => src.PublicationName))
.ForMember(dst => dst.PublicationYear, opt => opt.MapFrom(src => (DateTime?)(src.PublicationYear.HasValue ? new DateTime(src.PublicationYear.Value,1,1,0,0,0,DateTimeKind.Utc) : null)))
.ForMember(dst => dst.ReportingYear, opt =>opt.MapFrom(src => (DateTime?)(src.ReportingYear.HasValue ? new DateTime(src.ReportingYear.Value,1,1,0,0,0,DateTimeKind.Utc) : null)))
.ForMember(dst => dst.ApcPaymentYear, opt =>opt.MapFrom(src => (DateTime?)(src.ApcPaymentYear.HasValue ? new DateTime(src.ApcPaymentYear.Value,1,1,0,0,0,DateTimeKind.Utc) : null)))
.ForMember(dst => dst.PublicationYear, opt => opt.MapFrom(src => (DateTime?)(src.PublicationYear.HasValue ? new DateTime(src.PublicationYear.Value, 1, 1, 0, 0, 0, DateTimeKind.Utc) : null)))
.ForMember(dst => dst.ReportingYear, opt => opt.MapFrom(src => (DateTime?)(src.ReportingYear.HasValue ? new DateTime(src.ReportingYear.Value, 1, 1, 0, 0, 0, DateTimeKind.Utc) : null)))
.ForMember(dst => dst.ApcPaymentYear, opt => opt.MapFrom(src => (DateTime?)(src.ApcPaymentYear.HasValue ? new DateTime(src.ApcPaymentYear.Value, 1, 1, 0, 0, 0, DateTimeKind.Utc) : null)))
.ForMember(dst => dst.AuthorsText, opt => opt.MapFrom(src => src.AuthorsText))
.ForMember(dst => dst.DatabaseContributions, opt => opt.MapFrom(src => src.FactContributions))
.ForMember(dst => dst.OrgPublicationDatabaseContributionDTOs, opt => opt.MapFrom(src => src.InverseDimPublicationNavigation))
Expand All @@ -36,7 +37,7 @@ public PublicationProfile()
.ForMember(dst => dst.TargetAudience, opt => opt.MapFrom(src => src.TargetAudienceCodeNavigation))
.ForMember(dst => dst.Type, opt => opt.MapFrom(src => src.PublicationTypeCodeNavigation))
.ForMember(dst => dst.ThesisType, opt => opt.MapFrom(src => src.ThesisTypeCodeNavigation))
.ForMember(dst => dst.JournalName, opt => opt.MapFrom(src => src.JournalName))
.ForMember(dst => dst.JournalName, opt => opt.MapFrom(src => src.JournalName))
.ForMember(dst => dst.IssueNumber, opt => opt.MapFrom(src => src.IssueNumber))
.ForMember(dst => dst.ConferenceName, opt => opt.MapFrom(src => src.ConferenceName))
.ForMember(dst => dst.Volume, opt => opt.MapFrom(src => src.Volume))
Expand Down Expand Up @@ -73,7 +74,7 @@ public PublicationProfile()
.ForMember(dst => dst.SelfArchived, opt => opt.MapFrom(src => src.DimLocallyReportedPubInfos.Where(i => i.SelfArchivedType == SelfArchivedType)))
.ForMember(dst => dst.ArtPublicationTypeCategory, opt => opt.MapFrom(src => src.DimReferencedata))
.ForMember(dst => dst.OrgPublicationArtPublicatonTypeCategoryDTOs, opt => opt.MapFrom(src => src.InverseDimPublicationNavigation))
.ForMember(dst => dst.Abstract, opt => opt.MapFrom(src => src.Abstract))
.ForMember(dst => dst.Abstract, opt => opt.MapFrom(src => src.DimDescriptiveItems.Where(di => di.DescriptiveItemType == DescriptiveItemType_Abstract).Select(di => di.DescriptiveItem).FirstOrDefault()))
.ForMember(dst => dst.Created, opt => opt.MapFrom(src => src.Created))
.ForMember(dst => dst.Modified, opt => opt.MapFrom(src => src.Modified))
.ForMember(dst => dst.Doi, opt => opt.MapFrom(src => src.DimPids.Where(pid => pid.PidType == "doi").Select(pid => pid.PidContent).FirstOrDefault()))
Expand All @@ -85,19 +86,21 @@ public PublicationProfile()
.ForMember(dst => dst.ParentPublication, opt => opt.Ignore()) // Handled during in memory operations in the index repository
.ForMember(dst => dst.IsOrgPublication, opt => opt.MapFrom(src => src.DimPublicationId != null && src.DimPublicationId > 0)) // Publication is an organization publication, when DimPublicationId references co-publication. This property is used in query filter.
.ForMember(dst => dst.IsCoPublication, opt => opt.MapFrom(src => src.InverseDimPublicationNavigation.Count > 0)) // Publication is a co-publication, when InverseDimPublicationNavigation references one or more organization publications. This property is used in query filter.
.ForMember(dst => dst.CoPublicationID, opt => opt.MapFrom(src => src.DimPublicationId != null && src.DimPublicationId > 0 ? src.DimPublicationNavigation.PublicationId : null))
.ForMember(dst => dst.CoPublicationID, opt => opt.MapFrom(src => src.DimPublicationId != null && src.DimPublicationId > 0 ? src.DimPublicationNavigation.PublicationId : null))
.ForMember(dst => dst.OrgPublicationIDs, opt => opt.MapFrom(src => src.InverseDimPublicationNavigation.Select(t => t.PublicationId)))
.ForMember(dst => dst.OrganizationPartsOfCoPublication, opt => opt.MapFrom(src => src.InverseDimPublicationNavigation.Select(t =>
new OrganizationPartOfCoPublication() {
new OrganizationPartOfCoPublication()
{
Id = t.PublicationId,
OriginalPublicationId = t.OriginalPublicationId,
Organization = new OrganizationPartOfPublication_Organization() {
Organization = new OrganizationPartOfPublication_Organization()
{
Id = t.PublicationOrgId
}
}
)))
)))
.ForMember(dst => dst.ResearchfiUrl, opt => opt.Ignore()); // Handled during in memory operations in the index repository

CreateProjection<DimReferencedatum, ReferenceData>()
.AddTransform<string?>(s => string.IsNullOrWhiteSpace(s) ? null : s)
.ForMember(dst => dst.Code, opt => opt.MapFrom(src => src.CodeValue))
Expand All @@ -111,34 +114,34 @@ public PublicationProfile()
.ForMember(dst => dst.EmbargoDate, opt => opt.MapFrom(src => src.SelfArchivedEmbargoDate))
.ForMember(dst => dst.License, opt => opt.MapFrom(src => src.SelfArchivedLicenseCodeNavigation))
.ForMember(dst => dst.Version, opt => opt.MapFrom(src => src.SelfArchivedVersionCodeNavigation));

CreateProjection<DimKeyword, Keyword>()
.AddTransform<string?>(s => string.IsNullOrWhiteSpace(s) ? null : s)
.ForMember(dst => dst.Value, opt => opt.MapFrom(src => src.Keyword))
.ForMember(dst => dst.Language, opt => opt.MapFrom(src => src.Language))
.ForMember(dst => dst.Scheme, opt => opt.MapFrom(src => src.Scheme));

CreateProjection<DimFieldOfArt, ReferenceData>()
.AddTransform<string?>(s => string.IsNullOrWhiteSpace(s) ? null : s)
.ForMember(dst => dst.Code, opt => opt.MapFrom(src => src.FieldId))
.ForMember(dst => dst.NameFi, opt => opt.MapFrom(src => src.NameFi))
.ForMember(dst => dst.NameSv, opt => opt.MapFrom(src => src.NameSv))
.ForMember(dst => dst.NameEn, opt => opt.MapFrom(src => src.NameEn));

CreateProjection<DimFieldOfEducation, ReferenceData>()
.AddTransform<string?>(s => string.IsNullOrWhiteSpace(s) ? null : s)
.ForMember(dst => dst.Code, opt => opt.MapFrom(src => src.FieldId))
.ForMember(dst => dst.NameFi, opt => opt.MapFrom(src => src.NameFi))
.ForMember(dst => dst.NameSv, opt => opt.MapFrom(src => src.NameSv))
.ForMember(dst => dst.NameEn, opt => opt.MapFrom(src => src.NameEn));

CreateProjection<string, ReferenceData>()
.AddTransform<string?>(s => string.IsNullOrWhiteSpace(s) ? null : s)
.ForMember(dst => dst.Code, opt => opt.MapFrom(src => src))
.ForMember(dst => dst.NameFi, opt => opt.Ignore())
.ForMember(dst => dst.NameSv, opt => opt.Ignore())
.ForMember(dst => dst.NameEn, opt => opt.Ignore());

CreateProjection<int?, ReferenceData>()
.ForMember(dst => dst.Code, opt => opt.MapFrom(src => src.ToString()))
.ForMember(dst => dst.NameFi, opt => opt.Ignore())
Expand Down Expand Up @@ -177,10 +180,10 @@ public PublicationProfile()
.ForMember(dst => dst.FirstNames, opt => opt.MapFrom(src => src.FirstNames))
.ForMember(dst => dst.LastName, opt => opt.MapFrom(src => src.LastName))
.ForMember(dst => dst.Orcid,
opt => opt.MapFrom(src =>
opt => opt.MapFrom(src =>
src.DimKnownPersonIdConfirmedIdentityNavigation.DimPids
.FirstOrDefault(pid => pid.PidType == "orcid").PidContent));

CreateProjection<DimPid, PersistentIdentifier>()
.AddTransform<string?>(s => string.IsNullOrWhiteSpace(s) ? null : s)
.ForMember(dst => dst.Content, opt => opt.MapFrom(src => src.PidContent))
Expand Down
11 changes: 9 additions & 2 deletions aspnetcore/test/Indexer.Tests/Maps/PublicationProfileTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,14 @@ private static DimPublication GetEntity()
SelfArchivedType = "preprint"
}
},
Abstract = "abstract",
DimDescriptiveItems = new List<DimDescriptiveItem>
{
new()
{
DescriptiveItem = "test_abstract",
DescriptiveItemType = "abstract"
}
},
DimReferencedata = new List<DimReferencedatum>
{
new()
Expand Down Expand Up @@ -682,7 +689,7 @@ private Publication GetModel()
EmbargoDate = new DateTime(2023, 3, 10, 10, 40, 00)
}
},
Abstract = "abstract",
Abstract = "test_abstract",
ArtPublicationTypeCategory = new List<ReferenceData>
{
new()
Expand Down
Loading