diff --git a/.gitignore b/.gitignore index 8dfac7d0..42a1d5ae 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,12 @@ msbuild.wrn # Environment files /.env +# Merge conflict artifacts +*.BASE +*.LOCAL +*.REMOTE +*.bak + # Angular CLI and build artefacts /.angular-cli.json /.ng/ diff --git a/Artskart3.Api/Artskart3.Api.csproj b/Artskart3.Api/Artskart3.Api.csproj index a37fa826..b0e6f1ad 100644 --- a/Artskart3.Api/Artskart3.Api.csproj +++ b/Artskart3.Api/Artskart3.Api.csproj @@ -33,6 +33,7 @@ + diff --git a/Artskart3.Import.Core/Artskart3.Import.Core.csproj b/Artskart3.Import.Core/Artskart3.Import.Core.csproj new file mode 100644 index 00000000..9ed914b5 --- /dev/null +++ b/Artskart3.Import.Core/Artskart3.Import.Core.csproj @@ -0,0 +1,9 @@ + + + + net10.0 + enable + enable + + + diff --git a/Artskart3.Import.Core/Domain/Entities/DataSource.cs b/Artskart3.Import.Core/Domain/Entities/DataSource.cs new file mode 100644 index 00000000..4647915f --- /dev/null +++ b/Artskart3.Import.Core/Domain/Entities/DataSource.cs @@ -0,0 +1,93 @@ +using Artskart3.Import.Core.Domain.Enums; + +namespace Artskart3.Import.Core.Domain.Entities; + +/// +/// Provider configuration for a remote data source. +/// One flat table; use typed views or application-layer filtering to work with specific provider types. +/// +public class DataSource +{ + public int Id { get; set; } + + public string Name { get; set; } = string.Empty; + + /// + /// Discriminates the provider type and drives import pipeline branching. + /// + public ProviderType ProviderType { get; set; } + + /// + /// How many records to fetch per batch during harvest. + /// + public int RecordsBatchSize { get; set; } + + /// + /// Web service API version (ArtskartDataDeliveryWebServiceV1 providers). + /// + public int WebServiceVersion { get; set; } + + /// + /// URL of the remote endpoint (web service or IPT archive base URL). + /// + public string? RemoteAddress { get; set; } + + /// + /// IPT archive filename, or local cache filename for GbifApi providers. + /// + public string? ArchiveName { get; set; } + + /// + /// Internal notes for operators. + /// + public string? Notes { get; set; } + + public bool IsDeleted { get; set; } + + public bool IsEditedNameOrNotes { get; set; } + + /// + /// Flags records from this source as having non-valid occurrence IDs. + /// + public bool NonValidOccurrenceIds { get; set; } + + /// + /// JSON predicate string used by GbifApi providers to query the GBIF occurrence API. + /// + public string? GbifApiQuery { get; set; } + + /// + /// JSON overrides applied during the harvest phase (field mapping / transformations). + /// + public string? HarvestPropertyOverrides { get; set; } + + /// + /// JSON overrides applied during the import/processing phase. + /// + public string? ImportPropertyOverrides { get; set; } + + /// + /// How often (in days) the source should be re-harvested. + /// + public int UpdateFrequencyInDays { get; set; } + + /// + /// Digital Object Identifier — replaces GbifDataSetId from the old solution. + /// Required for active GbifIpt and GbifApi providers; enforced at the application layer. + /// + public string? Doi { get; set; } + + /// + /// GBIF publisher UUID. Used for API-based providers and publisher-scoped queries. + /// + public Guid? GbifPublisherId { get; set; } + + /// + /// Marks observations from this source as sensitive (restricted coordinates etc.). + /// + public bool IsSensitive { get; set; } + + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + public DateTime UpdatedAt { get; set; } = DateTime.UtcNow; + public DateTime? DeletedAt { get; set; } +} diff --git a/Artskart3.Import.Core/Domain/Entities/GbifDatasetDiscovery.cs b/Artskart3.Import.Core/Domain/Entities/GbifDatasetDiscovery.cs new file mode 100644 index 00000000..ebfb2157 --- /dev/null +++ b/Artskart3.Import.Core/Domain/Entities/GbifDatasetDiscovery.cs @@ -0,0 +1,57 @@ +using Artskart3.Import.Core.Domain.Enums; + +namespace Artskart3.Import.Core.Domain.Entities; + +public class GbifDatasetDiscovery +{ + public int Id { get; set; } + + /// + /// The GBIF dataset key (UUID). + /// + public string GbifId { get; set; } = string.Empty; + + /// + /// The GBIF publisher/organisation UUID. + /// + public Guid GbifPublisherId { get; set; } + + /// + /// The GBIF publishing organisation key string. + /// + public string PublishingOrganizationKey { get; set; } = string.Empty; + + public string Title { get; set; } = string.Empty; + + public string? Description { get; set; } + + /// + /// The dataset homepage URL on GBIF. + /// + public string? HomepageUrl { get; set; } + + /// + /// The Darwin Core Archive download URL. + /// + public string? DwcArchiveUrl { get; set; } + + public string? Doi { get; set; } + + /// + /// Additional identifiers for the dataset (stored as JSON array). + /// + public List Identifiers { get; set; } = []; + + /// + /// Review status of this discovered dataset. + /// + public DiscoveryStatus Status { get; set; } = DiscoveryStatus.Pending; + + /// + /// When this dataset was first discovered via the GBIF API. + /// + public DateTime DiscoveredAt { get; set; } + + public DateTime CreatedAt { get; set; } + public DateTime UpdatedAt { get; set; } +} diff --git a/Artskart3.Import.Core/Domain/Enums/DiscoveryStatus.cs b/Artskart3.Import.Core/Domain/Enums/DiscoveryStatus.cs new file mode 100644 index 00000000..d0197ec8 --- /dev/null +++ b/Artskart3.Import.Core/Domain/Enums/DiscoveryStatus.cs @@ -0,0 +1,19 @@ +namespace Artskart3.Import.Core.Domain.Enums; + +public enum DiscoveryStatus +{ + /// + /// Discovered via the GBIF API but not yet reviewed. + /// + Pending, + + /// + /// Reviewed and approved for import as a DataSource. + /// + Approved, + + /// + /// Reviewed and determined to be not relevant for import. + /// + Rejected +} diff --git a/Artskart3.Import.Core/Domain/Enums/ProviderType.cs b/Artskart3.Import.Core/Domain/Enums/ProviderType.cs new file mode 100644 index 00000000..ad1516b0 --- /dev/null +++ b/Artskart3.Import.Core/Domain/Enums/ProviderType.cs @@ -0,0 +1,9 @@ +namespace Artskart3.Import.Core.Domain.Enums; + +public enum ProviderType +{ + ArtskartDataDeliveryWebServiceV1 = 0, + GbifIpt = 1, + Artsobservasjoner20Api = 2, + GbifApi = 3 +} diff --git a/Artskart3.Import.Core/Domain/RepositoryInterfaces/IDataSourceRepository.cs b/Artskart3.Import.Core/Domain/RepositoryInterfaces/IDataSourceRepository.cs new file mode 100644 index 00000000..82e0be80 --- /dev/null +++ b/Artskart3.Import.Core/Domain/RepositoryInterfaces/IDataSourceRepository.cs @@ -0,0 +1,12 @@ +using Artskart3.Import.Core.Domain.Entities; + +namespace Artskart3.Import.Core.Domain.RepositoryInterfaces; + +public interface IDataSourceRepository +{ + Task> GetAllAsync(CancellationToken cancellationToken = default); + Task GetByIdAsync(int id, CancellationToken cancellationToken = default); + Task AddAsync(DataSource dataSource, CancellationToken cancellationToken = default); + Task UpdateAsync(DataSource dataSource, CancellationToken cancellationToken = default); + Task DeleteAsync(int id, CancellationToken cancellationToken = default); +} diff --git a/Artskart3.Import.Core/Domain/RepositoryInterfaces/IGbifDatasetDiscoveryRepository.cs b/Artskart3.Import.Core/Domain/RepositoryInterfaces/IGbifDatasetDiscoveryRepository.cs new file mode 100644 index 00000000..64919bf8 --- /dev/null +++ b/Artskart3.Import.Core/Domain/RepositoryInterfaces/IGbifDatasetDiscoveryRepository.cs @@ -0,0 +1,14 @@ +using Artskart3.Import.Core.Domain.Entities; +using Artskart3.Import.Core.Domain.Enums; + +namespace Artskart3.Import.Core.Domain.RepositoryInterfaces; + +public interface IGbifDatasetDiscoveryRepository +{ + Task> GetAllAsync(); + Task> GetByStatusAsync(DiscoveryStatus status); + Task GetByIdAsync(int id); + Task GetByGbifIdAsync(string gbifId); + Task AddAsync(GbifDatasetDiscovery discovery); + Task UpdateAsync(GbifDatasetDiscovery discovery); +} diff --git a/Artskart3.Import.Infrastructure/Artskart3.Import.Infrastructure.csproj b/Artskart3.Import.Infrastructure/Artskart3.Import.Infrastructure.csproj new file mode 100644 index 00000000..31beb36e --- /dev/null +++ b/Artskart3.Import.Infrastructure/Artskart3.Import.Infrastructure.csproj @@ -0,0 +1,29 @@ + + + + net10.0 + enable + enable + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + diff --git a/Artskart3.Import.Infrastructure/Data/ArtskartImportDbContext.cs b/Artskart3.Import.Infrastructure/Data/ArtskartImportDbContext.cs new file mode 100644 index 00000000..6e815f7a --- /dev/null +++ b/Artskart3.Import.Infrastructure/Data/ArtskartImportDbContext.cs @@ -0,0 +1,28 @@ +using Artskart3.Import.Core.Domain.Entities; +using Artskart3.Import.Infrastructure.Data.EntityConfigurations; +using Microsoft.EntityFrameworkCore; + +namespace Artskart3.Import.Infrastructure.Data; + +public class ArtskartImportDbContext : DbContext +{ + public ArtskartImportDbContext() + { + } + + public ArtskartImportDbContext(DbContextOptions options) + : base(options) + { + } + + public virtual DbSet DataSources { get; set; } + public virtual DbSet GbifDatasetDiscoveries { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.ApplyConfiguration(new DataSourceConfiguration()); + modelBuilder.ApplyConfiguration(new GbifDatasetDiscoveryConfiguration()); + + base.OnModelCreating(modelBuilder); + } +} diff --git a/Artskart3.Import.Infrastructure/Data/ArtskartImportDbContextFactory.cs b/Artskart3.Import.Infrastructure/Data/ArtskartImportDbContextFactory.cs new file mode 100644 index 00000000..caa622b4 --- /dev/null +++ b/Artskart3.Import.Infrastructure/Data/ArtskartImportDbContextFactory.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; +using Microsoft.Extensions.Configuration; + +namespace Artskart3.Import.Infrastructure.Data; + +/// +/// Used by EF Core tooling (dotnet ef migrations add, dotnet ef database update) at design time. +/// Reads the connection string from appsettings.json in the API project. +/// +public class ArtskartImportDbContextFactory : IDesignTimeDbContextFactory +{ + public ArtskartImportDbContext CreateDbContext(string[] args) + { + var configuration = new ConfigurationBuilder() + .SetBasePath(Path.Combine(Directory.GetCurrentDirectory(), "..", "Artskart3.Api")) + .AddJsonFile("appsettings.json", optional: false) + .AddJsonFile("appsettings.Development.json", optional: true) + .Build(); + + var connectionString = configuration.GetConnectionString("ArtskartImportDb") + ?? throw new InvalidOperationException("Connection string 'ArtskartImportDb' not found in appsettings.json."); + + var optionsBuilder = new DbContextOptionsBuilder(); + optionsBuilder.UseSqlServer(connectionString); + + return new ArtskartImportDbContext(optionsBuilder.Options); + } +} diff --git a/Artskart3.Import.Infrastructure/Data/EntityConfigurations/DataSourceConfiguration.cs b/Artskart3.Import.Infrastructure/Data/EntityConfigurations/DataSourceConfiguration.cs new file mode 100644 index 00000000..f61298df --- /dev/null +++ b/Artskart3.Import.Infrastructure/Data/EntityConfigurations/DataSourceConfiguration.cs @@ -0,0 +1,50 @@ +using Artskart3.Import.Core.Domain.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace Artskart3.Import.Infrastructure.Data.EntityConfigurations; + +public class DataSourceConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("DataSource"); + + builder.HasKey(e => e.Id); + + builder.Property(e => e.Name) + .IsRequired() + .HasMaxLength(500); + + builder.Property(e => e.ProviderType) + .IsRequired(); + + builder.Property(e => e.RemoteAddress) + .HasMaxLength(1000); + + builder.Property(e => e.ArchiveName) + .HasMaxLength(500); + + builder.Property(e => e.Notes) + .HasMaxLength(2000); + + // Large JSON columns — no length limit + builder.Property(e => e.GbifApiQuery) + .HasColumnType("nvarchar(max)"); + + builder.Property(e => e.HarvestPropertyOverrides) + .HasColumnType("nvarchar(max)"); + + builder.Property(e => e.ImportPropertyOverrides) + .HasColumnType("nvarchar(max)"); + + builder.Property(e => e.Doi) + .HasMaxLength(200); + + builder.HasIndex(e => e.ProviderType) + .HasDatabaseName("IX_DataSource_ProviderType"); + + builder.HasIndex(e => e.IsDeleted) + .HasDatabaseName("IX_DataSource_IsDeleted"); + } +} diff --git a/Artskart3.Import.Infrastructure/Data/EntityConfigurations/GbifDatasetDiscoveryConfiguration.cs b/Artskart3.Import.Infrastructure/Data/EntityConfigurations/GbifDatasetDiscoveryConfiguration.cs new file mode 100644 index 00000000..a6ef2df1 --- /dev/null +++ b/Artskart3.Import.Infrastructure/Data/EntityConfigurations/GbifDatasetDiscoveryConfiguration.cs @@ -0,0 +1,53 @@ +using Artskart3.Import.Core.Domain.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace Artskart3.Import.Infrastructure.Data.EntityConfigurations; + +public class GbifDatasetDiscoveryConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("GbifDatasetDiscovery"); + + builder.HasKey(e => e.Id); + + builder.Property(e => e.GbifId) + .IsRequired() + .HasMaxLength(200); + + builder.Property(e => e.PublishingOrganizationKey) + .IsRequired() + .HasMaxLength(200); + + builder.Property(e => e.Title) + .IsRequired() + .HasMaxLength(1000); + + builder.Property(e => e.Description) + .HasColumnType("nvarchar(max)"); + + builder.Property(e => e.HomepageUrl) + .HasMaxLength(1000); + + builder.Property(e => e.DwcArchiveUrl) + .HasMaxLength(1000); + + builder.Property(e => e.Doi) + .HasMaxLength(200); + + // Stored as a JSON array + builder.Property(e => e.Identifiers) + .HasColumnType("nvarchar(max)"); + + builder.Property(e => e.Status) + .IsRequired(); + + builder.HasIndex(e => e.GbifId) + .IsUnique() + .HasDatabaseName("IX_GbifDatasetDiscovery_GbifId"); + + builder.HasIndex(e => e.Status) + .HasDatabaseName("IX_GbifDatasetDiscovery_Status"); + } +} diff --git a/Artskart3.Import.Infrastructure/DependencyInjection/ImportServiceCollectionExtensions.cs b/Artskart3.Import.Infrastructure/DependencyInjection/ImportServiceCollectionExtensions.cs new file mode 100644 index 00000000..a8ab1002 --- /dev/null +++ b/Artskart3.Import.Infrastructure/DependencyInjection/ImportServiceCollectionExtensions.cs @@ -0,0 +1,23 @@ +using Artskart3.Import.Core.Domain.RepositoryInterfaces; +using Artskart3.Import.Infrastructure.Data; +using Artskart3.Import.Infrastructure.Persistence.Repositories; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; + +namespace Artskart3.Import.Infrastructure.DependencyInjection; + +public static class ImportServiceCollectionExtensions +{ + public static IServiceCollection AddImportInfrastructure( + this IServiceCollection services, + string connectionString) + { + services.AddDbContext(options => + options.UseSqlServer(connectionString)); + + services.AddScoped(); + services.AddScoped(); + + return services; + } +} diff --git a/Artskart3.Import.Infrastructure/Migrations/20260430124749_InitialImportBaseline.Designer.cs b/Artskart3.Import.Infrastructure/Migrations/20260430124749_InitialImportBaseline.Designer.cs new file mode 100644 index 00000000..971a921d --- /dev/null +++ b/Artskart3.Import.Infrastructure/Migrations/20260430124749_InitialImportBaseline.Designer.cs @@ -0,0 +1,115 @@ +// +using System; +using Artskart3.Import.Infrastructure.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace Artskart3.Import.Infrastructure.Migrations +{ + [DbContext(typeof(ArtskartImportDbContext))] + [Migration("20260430124749_InitialImportBaseline")] + partial class InitialImportBaseline + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.5") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("Artskart3.Import.Core.Domain.Entities.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ArchiveName") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("DeletedAt") + .HasColumnType("datetime2"); + + b.Property("Doi") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("GbifApiQuery") + .HasColumnType("nvarchar(max)"); + + b.Property("GbifPublisherId") + .HasColumnType("uniqueidentifier"); + + b.Property("HarvestPropertyOverrides") + .HasColumnType("nvarchar(max)"); + + b.Property("ImportPropertyOverrides") + .HasColumnType("nvarchar(max)"); + + b.Property("IsDeleted") + .HasColumnType("bit"); + + b.Property("IsEditedNameOrNotes") + .HasColumnType("bit"); + + b.Property("IsSensitive") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("NonValidOccurrenceIds") + .HasColumnType("bit"); + + b.Property("Notes") + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)"); + + b.Property("ProviderType") + .HasColumnType("int"); + + b.Property("RecordsBatchSize") + .HasColumnType("int"); + + b.Property("RemoteAddress") + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.Property("UpdateFrequencyInDays") + .HasColumnType("int"); + + b.Property("UpdatedAt") + .HasColumnType("datetime2"); + + b.Property("WebServiceVersion") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("IsDeleted") + .HasDatabaseName("IX_DataSource_IsDeleted"); + + b.HasIndex("ProviderType") + .HasDatabaseName("IX_DataSource_ProviderType"); + + b.ToTable("DataSource", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Artskart3.Import.Infrastructure/Migrations/20260430124749_InitialImportBaseline.cs b/Artskart3.Import.Infrastructure/Migrations/20260430124749_InitialImportBaseline.cs new file mode 100644 index 00000000..2c019ee3 --- /dev/null +++ b/Artskart3.Import.Infrastructure/Migrations/20260430124749_InitialImportBaseline.cs @@ -0,0 +1,64 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Artskart3.Import.Infrastructure.Migrations +{ + /// + public partial class InitialImportBaseline : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "DataSource", + columns: table => new + { + Id = table.Column(type: "int", nullable: false) + .Annotation("SqlServer:Identity", "1, 1"), + Name = table.Column(type: "nvarchar(500)", maxLength: 500, nullable: false), + ProviderType = table.Column(type: "int", nullable: false), + RecordsBatchSize = table.Column(type: "int", nullable: false), + WebServiceVersion = table.Column(type: "int", nullable: false), + RemoteAddress = table.Column(type: "nvarchar(1000)", maxLength: 1000, nullable: true), + ArchiveName = table.Column(type: "nvarchar(500)", maxLength: 500, nullable: true), + Notes = table.Column(type: "nvarchar(2000)", maxLength: 2000, nullable: true), + IsDeleted = table.Column(type: "bit", nullable: false), + IsEditedNameOrNotes = table.Column(type: "bit", nullable: false), + NonValidOccurrenceIds = table.Column(type: "bit", nullable: false), + GbifApiQuery = table.Column(type: "nvarchar(max)", nullable: true), + HarvestPropertyOverrides = table.Column(type: "nvarchar(max)", nullable: true), + ImportPropertyOverrides = table.Column(type: "nvarchar(max)", nullable: true), + UpdateFrequencyInDays = table.Column(type: "int", nullable: false), + Doi = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: true), + GbifPublisherId = table.Column(type: "uniqueidentifier", nullable: true), + IsSensitive = table.Column(type: "bit", nullable: false), + CreatedAt = table.Column(type: "datetime2", nullable: false), + UpdatedAt = table.Column(type: "datetime2", nullable: false), + DeletedAt = table.Column(type: "datetime2", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_DataSource", x => x.Id); + }); + + migrationBuilder.CreateIndex( + name: "IX_DataSource_IsDeleted", + table: "DataSource", + column: "IsDeleted"); + + migrationBuilder.CreateIndex( + name: "IX_DataSource_ProviderType", + table: "DataSource", + column: "ProviderType"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "DataSource"); + } + } +} diff --git a/Artskart3.Import.Infrastructure/Migrations/20260505115624_AddGbifDatasetDiscovery.Designer.cs b/Artskart3.Import.Infrastructure/Migrations/20260505115624_AddGbifDatasetDiscovery.Designer.cs new file mode 100644 index 00000000..f99217db --- /dev/null +++ b/Artskart3.Import.Infrastructure/Migrations/20260505115624_AddGbifDatasetDiscovery.Designer.cs @@ -0,0 +1,184 @@ +// +using System; +using Artskart3.Import.Infrastructure.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace Artskart3.Import.Infrastructure.Migrations +{ + [DbContext(typeof(ArtskartImportDbContext))] + [Migration("20260505115624_AddGbifDatasetDiscovery")] + partial class AddGbifDatasetDiscovery + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.5") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("Artskart3.Import.Core.Domain.Entities.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ArchiveName") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("DeletedAt") + .HasColumnType("datetime2"); + + b.Property("Doi") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("GbifApiQuery") + .HasColumnType("nvarchar(max)"); + + b.Property("GbifPublisherId") + .HasColumnType("uniqueidentifier"); + + b.Property("HarvestPropertyOverrides") + .HasColumnType("nvarchar(max)"); + + b.Property("ImportPropertyOverrides") + .HasColumnType("nvarchar(max)"); + + b.Property("IsDeleted") + .HasColumnType("bit"); + + b.Property("IsEditedNameOrNotes") + .HasColumnType("bit"); + + b.Property("IsSensitive") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("NonValidOccurrenceIds") + .HasColumnType("bit"); + + b.Property("Notes") + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)"); + + b.Property("ProviderType") + .HasColumnType("int"); + + b.Property("RecordsBatchSize") + .HasColumnType("int"); + + b.Property("RemoteAddress") + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.Property("UpdateFrequencyInDays") + .HasColumnType("int"); + + b.Property("UpdatedAt") + .HasColumnType("datetime2"); + + b.Property("WebServiceVersion") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("IsDeleted") + .HasDatabaseName("IX_DataSource_IsDeleted"); + + b.HasIndex("ProviderType") + .HasDatabaseName("IX_DataSource_ProviderType"); + + b.ToTable("DataSource", (string)null); + }); + + modelBuilder.Entity("Artskart3.Import.Core.Domain.Entities.GbifDatasetDiscovery", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("Description") + .HasColumnType("nvarchar(max)"); + + b.Property("DiscoveredAt") + .HasColumnType("datetime2"); + + b.Property("Doi") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("DwcArchiveUrl") + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.Property("GbifId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("GbifPublisherId") + .HasColumnType("uniqueidentifier"); + + b.Property("HomepageUrl") + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.PrimitiveCollection("Identifiers") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("PublishingOrganizationKey") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.Property("UpdatedAt") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("GbifId") + .IsUnique() + .HasDatabaseName("IX_GbifDatasetDiscovery_GbifId"); + + b.HasIndex("Status") + .HasDatabaseName("IX_GbifDatasetDiscovery_Status"); + + b.ToTable("GbifDatasetDiscovery", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Artskart3.Import.Infrastructure/Migrations/20260505115624_AddGbifDatasetDiscovery.cs b/Artskart3.Import.Infrastructure/Migrations/20260505115624_AddGbifDatasetDiscovery.cs new file mode 100644 index 00000000..c004028f --- /dev/null +++ b/Artskart3.Import.Infrastructure/Migrations/20260505115624_AddGbifDatasetDiscovery.cs @@ -0,0 +1,58 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Artskart3.Import.Infrastructure.Migrations +{ + /// + public partial class AddGbifDatasetDiscovery : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "GbifDatasetDiscovery", + columns: table => new + { + Id = table.Column(type: "int", nullable: false) + .Annotation("SqlServer:Identity", "1, 1"), + GbifId = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: false), + GbifPublisherId = table.Column(type: "uniqueidentifier", nullable: false), + PublishingOrganizationKey = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: false), + Title = table.Column(type: "nvarchar(1000)", maxLength: 1000, nullable: false), + Description = table.Column(type: "nvarchar(max)", nullable: true), + HomepageUrl = table.Column(type: "nvarchar(1000)", maxLength: 1000, nullable: true), + DwcArchiveUrl = table.Column(type: "nvarchar(1000)", maxLength: 1000, nullable: true), + Doi = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: true), + Identifiers = table.Column(type: "nvarchar(max)", nullable: false), + Status = table.Column(type: "int", nullable: false), + DiscoveredAt = table.Column(type: "datetime2", nullable: false), + CreatedAt = table.Column(type: "datetime2", nullable: false), + UpdatedAt = table.Column(type: "datetime2", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_GbifDatasetDiscovery", x => x.Id); + }); + + migrationBuilder.CreateIndex( + name: "IX_GbifDatasetDiscovery_GbifId", + table: "GbifDatasetDiscovery", + column: "GbifId", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_GbifDatasetDiscovery_Status", + table: "GbifDatasetDiscovery", + column: "Status"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "GbifDatasetDiscovery"); + } + } +} diff --git a/Artskart3.Import.Infrastructure/Migrations/ArtskartImportDbContextModelSnapshot.cs b/Artskart3.Import.Infrastructure/Migrations/ArtskartImportDbContextModelSnapshot.cs new file mode 100644 index 00000000..40a07710 --- /dev/null +++ b/Artskart3.Import.Infrastructure/Migrations/ArtskartImportDbContextModelSnapshot.cs @@ -0,0 +1,181 @@ +// +using System; +using Artskart3.Import.Infrastructure.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace Artskart3.Import.Infrastructure.Migrations +{ + [DbContext(typeof(ArtskartImportDbContext))] + partial class ArtskartImportDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.5") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("Artskart3.Import.Core.Domain.Entities.DataSource", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("ArchiveName") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("DeletedAt") + .HasColumnType("datetime2"); + + b.Property("Doi") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("GbifApiQuery") + .HasColumnType("nvarchar(max)"); + + b.Property("GbifPublisherId") + .HasColumnType("uniqueidentifier"); + + b.Property("HarvestPropertyOverrides") + .HasColumnType("nvarchar(max)"); + + b.Property("ImportPropertyOverrides") + .HasColumnType("nvarchar(max)"); + + b.Property("IsDeleted") + .HasColumnType("bit"); + + b.Property("IsEditedNameOrNotes") + .HasColumnType("bit"); + + b.Property("IsSensitive") + .HasColumnType("bit"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("NonValidOccurrenceIds") + .HasColumnType("bit"); + + b.Property("Notes") + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)"); + + b.Property("ProviderType") + .HasColumnType("int"); + + b.Property("RecordsBatchSize") + .HasColumnType("int"); + + b.Property("RemoteAddress") + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.Property("UpdateFrequencyInDays") + .HasColumnType("int"); + + b.Property("UpdatedAt") + .HasColumnType("datetime2"); + + b.Property("WebServiceVersion") + .HasColumnType("int"); + + b.HasKey("Id"); + + b.HasIndex("IsDeleted") + .HasDatabaseName("IX_DataSource_IsDeleted"); + + b.HasIndex("ProviderType") + .HasDatabaseName("IX_DataSource_ProviderType"); + + b.ToTable("DataSource", (string)null); + }); + + modelBuilder.Entity("Artskart3.Import.Core.Domain.Entities.GbifDatasetDiscovery", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("int"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedAt") + .HasColumnType("datetime2"); + + b.Property("Description") + .HasColumnType("nvarchar(max)"); + + b.Property("DiscoveredAt") + .HasColumnType("datetime2"); + + b.Property("Doi") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("DwcArchiveUrl") + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.Property("GbifId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("GbifPublisherId") + .HasColumnType("uniqueidentifier"); + + b.Property("HomepageUrl") + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.PrimitiveCollection("Identifiers") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("PublishingOrganizationKey") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("Title") + .IsRequired() + .HasMaxLength(1000) + .HasColumnType("nvarchar(1000)"); + + b.Property("UpdatedAt") + .HasColumnType("datetime2"); + + b.HasKey("Id"); + + b.HasIndex("GbifId") + .IsUnique() + .HasDatabaseName("IX_GbifDatasetDiscovery_GbifId"); + + b.HasIndex("Status") + .HasDatabaseName("IX_GbifDatasetDiscovery_Status"); + + b.ToTable("GbifDatasetDiscovery", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/Artskart3.Import.Infrastructure/Persistence/Repositories/DataSourceRepository.cs b/Artskart3.Import.Infrastructure/Persistence/Repositories/DataSourceRepository.cs new file mode 100644 index 00000000..ecdb0900 --- /dev/null +++ b/Artskart3.Import.Infrastructure/Persistence/Repositories/DataSourceRepository.cs @@ -0,0 +1,57 @@ +using Artskart3.Import.Core.Domain.Entities; +using Artskart3.Import.Core.Domain.RepositoryInterfaces; +using Artskart3.Import.Infrastructure.Data; +using Microsoft.EntityFrameworkCore; + +namespace Artskart3.Import.Infrastructure.Persistence.Repositories; + +public class DataSourceRepository : IDataSourceRepository +{ + private readonly ArtskartImportDbContext _context; + + public DataSourceRepository(ArtskartImportDbContext context) + { + _context = context; + } + + public async Task> GetAllAsync(CancellationToken cancellationToken = default) + { + return await _context.DataSources + .Where(d => !d.IsDeleted) + .OrderBy(d => d.Name) + .ToListAsync(cancellationToken); + } + + public async Task GetByIdAsync(int id, CancellationToken cancellationToken = default) + { + return await _context.DataSources + .FirstOrDefaultAsync(d => d.Id == id, cancellationToken); + } + + public async Task AddAsync(DataSource dataSource, CancellationToken cancellationToken = default) + { + dataSource.CreatedAt = DateTime.UtcNow; + dataSource.UpdatedAt = DateTime.UtcNow; + _context.DataSources.Add(dataSource); + await _context.SaveChangesAsync(cancellationToken); + return dataSource; + } + + public async Task UpdateAsync(DataSource dataSource, CancellationToken cancellationToken = default) + { + dataSource.UpdatedAt = DateTime.UtcNow; + _context.DataSources.Update(dataSource); + await _context.SaveChangesAsync(cancellationToken); + } + + public async Task DeleteAsync(int id, CancellationToken cancellationToken = default) + { + var dataSource = await GetByIdAsync(id, cancellationToken); + if (dataSource is null) return; + + dataSource.IsDeleted = true; + dataSource.DeletedAt = DateTime.UtcNow; + dataSource.UpdatedAt = DateTime.UtcNow; + await _context.SaveChangesAsync(cancellationToken); + } +} diff --git a/Artskart3.Import.Infrastructure/Persistence/Repositories/GbifDatasetDiscoveryRepository.cs b/Artskart3.Import.Infrastructure/Persistence/Repositories/GbifDatasetDiscoveryRepository.cs new file mode 100644 index 00000000..51581f95 --- /dev/null +++ b/Artskart3.Import.Infrastructure/Persistence/Repositories/GbifDatasetDiscoveryRepository.cs @@ -0,0 +1,44 @@ +using Artskart3.Import.Core.Domain.Entities; +using Artskart3.Import.Core.Domain.Enums; +using Artskart3.Import.Core.Domain.RepositoryInterfaces; +using Artskart3.Import.Infrastructure.Data; +using Microsoft.EntityFrameworkCore; + +namespace Artskart3.Import.Infrastructure.Persistence.Repositories; + +public class GbifDatasetDiscoveryRepository : IGbifDatasetDiscoveryRepository +{ + private readonly ArtskartImportDbContext _context; + + public GbifDatasetDiscoveryRepository(ArtskartImportDbContext context) + { + _context = context; + } + + public async Task> GetAllAsync() + => await _context.GbifDatasetDiscoveries.ToListAsync(); + + public async Task> GetByStatusAsync(DiscoveryStatus status) + => await _context.GbifDatasetDiscoveries.Where(d => d.Status == status).ToListAsync(); + + public async Task GetByIdAsync(int id) + => await _context.GbifDatasetDiscoveries.FindAsync(id); + + public async Task GetByGbifIdAsync(string gbifId) + => await _context.GbifDatasetDiscoveries.FirstOrDefaultAsync(d => d.GbifId == gbifId); + + public async Task AddAsync(GbifDatasetDiscovery discovery) + { + discovery.CreatedAt = DateTime.UtcNow; + discovery.UpdatedAt = DateTime.UtcNow; + await _context.GbifDatasetDiscoveries.AddAsync(discovery); + await _context.SaveChangesAsync(); + } + + public async Task UpdateAsync(GbifDatasetDiscovery discovery) + { + discovery.UpdatedAt = DateTime.UtcNow; + _context.GbifDatasetDiscoveries.Update(discovery); + await _context.SaveChangesAsync(); + } +} diff --git a/Artskart3.slnx b/Artskart3.slnx index bb569d2a..a3704048 100644 --- a/Artskart3.slnx +++ b/Artskart3.slnx @@ -9,4 +9,6 @@ + + diff --git a/README.md b/README.md index 65dd9f90..c4628afb 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,50 @@ dotnet ef migrations remove --startup-project ..\Artskart3.Api > Migrasjonsfilene ligger i `Artskart3.Infrastructure/Migrations/`. Ikke rediger disse manuelt etter at de er kjørt mot en delt database. +### Import-databasen (Artskart3Import) + +Import-laget følger Clean Architecture og er delt i to prosjekter: +- `Artskart3.Import.Core` — domeneentiteter og repository-grensesnitt (ingen EF-avhengigheter) +- `Artskart3.Import.Infrastructure` — EF DbContext, migrasjoner og repository-implementasjoner + +**Opprett/oppdater lokal import-database:** +```powershell +dotnet ef database update ` + --project Artskart3.Import.Infrastructure ` + --startup-project Artskart3.Api ` + --context ArtskartImportDbContext +``` + +**Legge til en ny migrasjon for import-konteksten:** +```powershell +dotnet ef migrations add ` + --project Artskart3.Import.Infrastructure ` + --startup-project Artskart3.Api ` + --context ArtskartImportDbContext +``` + +**Angre siste import-migrasjon:** +```powershell +dotnet ef migrations remove ` + --project Artskart3.Import.Infrastructure ` + --startup-project Artskart3.Api ` + --context ArtskartImportDbContext +``` + +> Migrasjonsfilene ligger i `Artskart3.Import.Infrastructure/Migrations/`. + +### Databaseoversikt + +Løsningen bruker tre separate databaser: + +| Database | Prosjekt | Formål | +|---|---|---| +| `Artskart3Search` | `Artskart3.Infrastructure` | Søkeoptimalisert leseindeks | +| `Artskart3Import` | `Artskart3.Import.Infrastructure` | Rå leverandørdata fra eksterne kilder (DwC-A, GBIF API m.m.) | +| `Artskart3Operational` | Kommer | Validerte og prosesserte forekomstdata (kilde til sannhet) | + +Taxonomidata hentes fra [Nortaxa](https://nortaxa.artsdatabanken.no) via API og caches lokalt i `Artskart3Search`. + ## Navngiving av branches Standariserer navngiving av branches er `feature/navn-på-branch` som for eksempel: `feature/authentication` for features og `bugfix/fix-ip-blocking` hvis det er en bugfix.