138 datasource entity ef migration

.gitignore

@@ -36,6 +36,12 @@ msbuild.wrn
 # Environment files
 /.env
 
+# Merge conflict artifacts
+*.BASE
+*.LOCAL
+*.REMOTE
+*.bak
+
 # Angular CLI and build artefacts
 /.angular-cli.json
 /.ng/

Artskart3.Api/Artskart3.Api.csproj

@@ -33,6 +33,7 @@
   <ItemGroup>
     <ProjectReference Include="..\Artskart3.Core\Artskart3.Core.csproj" />
     <ProjectReference Include="..\Artskart3.Infrastructure\Artskart3.Infrastructure.csproj" />
+    <ProjectReference Include="..\Artskart3.Import.Infrastructure\Artskart3.Import.Infrastructure.csproj" />
   </ItemGroup>
 
 </Project>

Artskart3.Import.Core/Artskart3.Import.Core.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

Artskart3.Import.Core/Domain/Entities/DataSource.cs

@@ -0,0 +1,93 @@
+using Artskart3.Import.Core.Domain.Enums;
+
+namespace Artskart3.Import.Core.Domain.Entities;
+
+/// <summary>
+/// Provider configuration for a remote data source.
+/// One flat table; use typed views or application-layer filtering to work with specific provider types.
+/// </summary>
+public class DataSource
+{
+    public int Id { get; set; }
+
+    public string Name { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Discriminates the provider type and drives import pipeline branching.
+    /// </summary>
+    public ProviderType ProviderType { get; set; }
+
+    /// <summary>
+    /// How many records to fetch per batch during harvest.
+    /// </summary>
+    public int RecordsBatchSize { get; set; }
+
+    /// <summary>
+    /// Web service API version (ArtskartDataDeliveryWebServiceV1 providers).
+    /// </summary>
+    public int WebServiceVersion { get; set; }
+
+    /// <summary>
+    /// URL of the remote endpoint (web service or IPT archive base URL).
+    /// </summary>
+    public string? RemoteAddress { get; set; }
+
+    /// <summary>
+    /// IPT archive filename, or local cache filename for GbifApi providers.
+    /// </summary>
+    public string? ArchiveName { get; set; }
+
+    /// <summary>
+    /// Internal notes for operators.
+    /// </summary>
+    public string? Notes { get; set; }
+
+    public bool IsDeleted { get; set; }
+
+    public bool IsEditedNameOrNotes { get; set; }
+
+    /// <summary>
+    /// Flags records from this source as having non-valid occurrence IDs.
+    /// </summary>
+    public bool NonValidOccurrenceIds { get; set; }
+
+    /// <summary>
+    /// JSON predicate string used by GbifApi providers to query the GBIF occurrence API.
+    /// </summary>
+    public string? GbifApiQuery { get; set; }
+
+    /// <summary>
+    /// JSON overrides applied during the harvest phase (field mapping / transformations).
+    /// </summary>
+    public string? HarvestPropertyOverrides { get; set; }
+
+    /// <summary>
+    /// JSON overrides applied during the import/processing phase.
+    /// </summary>
+    public string? ImportPropertyOverrides { get; set; }
+
+    /// <summary>
+    /// How often (in days) the source should be re-harvested.
+    /// </summary>
+    public int UpdateFrequencyInDays { get; set; }
+
+    /// <summary>
+    /// Digital Object Identifier — replaces GbifDataSetId from the old solution.
+    /// Required for active GbifIpt and GbifApi providers; enforced at the application layer.
+    /// </summary>
+    public string? Doi { get; set; }
+
+    /// <summary>
+    /// GBIF publisher UUID. Used for API-based providers and publisher-scoped queries.
+    /// </summary>
+    public Guid? GbifPublisherId { get; set; }
+
+    /// <summary>
+    /// Marks observations from this source as sensitive (restricted coordinates etc.).
+    /// </summary>
+    public bool IsSensitive { get; set; }
+
+    public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
+    public DateTime UpdatedAt { get; set; } = DateTime.UtcNow;
+    public DateTime? DeletedAt { get; set; }
+}

Artskart3.Import.Core/Domain/Entities/GbifDatasetDiscovery.cs

@@ -0,0 +1,57 @@
+using Artskart3.Import.Core.Domain.Enums;
+
+namespace Artskart3.Import.Core.Domain.Entities;
+
+public class GbifDatasetDiscovery
+{
+    public int Id { get; set; }
+
+    /// <summary>
+    /// The GBIF dataset key (UUID).
+    /// </summary>
+    public string GbifId { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The GBIF publisher/organisation UUID.
+    /// </summary>
+    public Guid GbifPublisherId { get; set; }
+
+    /// <summary>
+    /// The GBIF publishing organisation key string.
+    /// </summary>
+    public string PublishingOrganizationKey { get; set; } = string.Empty;
+
+    public string Title { get; set; } = string.Empty;
+
+    public string? Description { get; set; }
+
+    /// <summary>
+    /// The dataset homepage URL on GBIF.
+    /// </summary>
+    public string? HomepageUrl { get; set; }
+
+    /// <summary>
+    /// The Darwin Core Archive download URL.
+    /// </summary>
+    public string? DwcArchiveUrl { get; set; }
+
+    public string? Doi { get; set; }
+
+    /// <summary>
+    /// Additional identifiers for the dataset (stored as JSON array).
+    /// </summary>
+    public List<string> Identifiers { get; set; } = [];
+
+    /// <summary>
+    /// Review status of this discovered dataset.
+    /// </summary>
+    public DiscoveryStatus Status { get; set; } = DiscoveryStatus.Pending;
+
+    /// <summary>
+    /// When this dataset was first discovered via the GBIF API.
+    /// </summary>
+    public DateTime DiscoveredAt { get; set; }
+
+    public DateTime CreatedAt { get; set; }
+    public DateTime UpdatedAt { get; set; }
+}

Artskart3.Import.Core/Domain/Enums/DiscoveryStatus.cs

@@ -0,0 +1,19 @@
+namespace Artskart3.Import.Core.Domain.Enums;
+
+public enum DiscoveryStatus
+{
+    /// <summary>
+    /// Discovered via the GBIF API but not yet reviewed.
+    /// </summary>
+    Pending,
+
+    /// <summary>
+    /// Reviewed and approved for import as a DataSource.
+    /// </summary>
+    Approved,
+
+    /// <summary>
+    /// Reviewed and determined to be not relevant for import.
+    /// </summary>
+    Rejected
+}

Artskart3.Import.Core/Domain/Enums/ProviderType.cs

@@ -0,0 +1,9 @@
+namespace Artskart3.Import.Core.Domain.Enums;
+
+public enum ProviderType
+{
+    ArtskartDataDeliveryWebServiceV1 = 0,
+    GbifIpt = 1,
+    Artsobservasjoner20Api = 2,
+    GbifApi = 3
+}

Artskart3.Import.Core/Domain/RepositoryInterfaces/IDataSourceRepository.cs

@@ -0,0 +1,12 @@
+using Artskart3.Import.Core.Domain.Entities;
+
+namespace Artskart3.Import.Core.Domain.RepositoryInterfaces;
+
+public interface IDataSourceRepository
+{
+    Task<IEnumerable<DataSource>> GetAllAsync(CancellationToken cancellationToken = default);
+    Task<DataSource?> GetByIdAsync(int id, CancellationToken cancellationToken = default);
+    Task<DataSource> AddAsync(DataSource dataSource, CancellationToken cancellationToken = default);
+    Task UpdateAsync(DataSource dataSource, CancellationToken cancellationToken = default);
+    Task DeleteAsync(int id, CancellationToken cancellationToken = default);
+}

Artskart3.Import.Core/Domain/RepositoryInterfaces/IGbifDatasetDiscoveryRepository.cs

@@ -0,0 +1,14 @@
+using Artskart3.Import.Core.Domain.Entities;
+using Artskart3.Import.Core.Domain.Enums;
+
+namespace Artskart3.Import.Core.Domain.RepositoryInterfaces;
+
+public interface IGbifDatasetDiscoveryRepository
+{
+    Task<IEnumerable<GbifDatasetDiscovery>> GetAllAsync();
+    Task<IEnumerable<GbifDatasetDiscovery>> GetByStatusAsync(DiscoveryStatus status);
+    Task<GbifDatasetDiscovery?> GetByIdAsync(int id);
+    Task<GbifDatasetDiscovery?> GetByGbifIdAsync(string gbifId);
+    Task AddAsync(GbifDatasetDiscovery discovery);
+    Task UpdateAsync(GbifDatasetDiscovery discovery);
+}

Artskart3.Import.Infrastructure/Artskart3.Import.Infrastructure.csproj

@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Artskart3.Import.Core\Artskart3.Import.Core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="10.0.5" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.5">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="10.0.5" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Tools" Version="10.0.5">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="10.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="10.0.0" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.5" />
+  </ItemGroup>
+
+</Project>

Artskart3.Import.Infrastructure/Data/ArtskartImportDbContext.cs

@@ -0,0 +1,28 @@
+using Artskart3.Import.Core.Domain.Entities;
+using Artskart3.Import.Infrastructure.Data.EntityConfigurations;
+using Microsoft.EntityFrameworkCore;
+
+namespace Artskart3.Import.Infrastructure.Data;
+
+public class ArtskartImportDbContext : DbContext
+{
+    public ArtskartImportDbContext()
+    {
+    }
+
+    public ArtskartImportDbContext(DbContextOptions<ArtskartImportDbContext> options)
+        : base(options)
+    {
+    }
+
+    public virtual DbSet<DataSource> DataSources { get; set; }
+    public virtual DbSet<GbifDatasetDiscovery> GbifDatasetDiscoveries { get; set; }
+
+    protected override void OnModelCreating(ModelBuilder modelBuilder)
+    {
+        modelBuilder.ApplyConfiguration(new DataSourceConfiguration());
+        modelBuilder.ApplyConfiguration(new GbifDatasetDiscoveryConfiguration());
+
+        base.OnModelCreating(modelBuilder);
+    }
+}

Artskart3.Import.Infrastructure/Data/ArtskartImportDbContextFactory.cs

@@ -0,0 +1,29 @@
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Design;
+using Microsoft.Extensions.Configuration;
+
+namespace Artskart3.Import.Infrastructure.Data;
+
+/// <summary>
+/// Used by EF Core tooling (dotnet ef migrations add, dotnet ef database update) at design time.
+/// Reads the connection string from appsettings.json in the API project.
+/// </summary>
+public class ArtskartImportDbContextFactory : IDesignTimeDbContextFactory<ArtskartImportDbContext>
+{
+    public ArtskartImportDbContext CreateDbContext(string[] args)
+    {
+        var configuration = new ConfigurationBuilder()
+            .SetBasePath(Path.Combine(Directory.GetCurrentDirectory(), "..", "Artskart3.Api"))
+            .AddJsonFile("appsettings.json", optional: false)
+            .AddJsonFile("appsettings.Development.json", optional: true)
+            .Build();
+
+        var connectionString = configuration.GetConnectionString("ArtskartImportDb")
+            ?? throw new InvalidOperationException("Connection string 'ArtskartImportDb' not found in appsettings.json.");
+
+        var optionsBuilder = new DbContextOptionsBuilder<ArtskartImportDbContext>();
+        optionsBuilder.UseSqlServer(connectionString);
+
+        return new ArtskartImportDbContext(optionsBuilder.Options);
+    }
+}

Artskart3.Import.Infrastructure/Data/EntityConfigurations/DataSourceConfiguration.cs

@@ -0,0 +1,50 @@
+using Artskart3.Import.Core.Domain.Entities;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Metadata.Builders;
+
+namespace Artskart3.Import.Infrastructure.Data.EntityConfigurations;
+
+public class DataSourceConfiguration : IEntityTypeConfiguration<DataSource>
+{
+    public void Configure(EntityTypeBuilder<DataSource> builder)
+    {
+        builder.ToTable("DataSource");
+
+        builder.HasKey(e => e.Id);
+
+        builder.Property(e => e.Name)
+            .IsRequired()
+            .HasMaxLength(500);
+
+        builder.Property(e => e.ProviderType)
+            .IsRequired();
+
+        builder.Property(e => e.RemoteAddress)
+            .HasMaxLength(1000);
+
+        builder.Property(e => e.ArchiveName)
+            .HasMaxLength(500);
+
+        builder.Property(e => e.Notes)
+            .HasMaxLength(2000);
+
+        // Large JSON columns — no length limit
+        builder.Property(e => e.GbifApiQuery)
+            .HasColumnType("nvarchar(max)");
+
+        builder.Property(e => e.HarvestPropertyOverrides)
+            .HasColumnType("nvarchar(max)");
+
+        builder.Property(e => e.ImportPropertyOverrides)
+            .HasColumnType("nvarchar(max)");
+
+        builder.Property(e => e.Doi)
+            .HasMaxLength(200);
+
+        builder.HasIndex(e => e.ProviderType)
+            .HasDatabaseName("IX_DataSource_ProviderType");
+
+        builder.HasIndex(e => e.IsDeleted)
+            .HasDatabaseName("IX_DataSource_IsDeleted");
+    }
+}