Skip to content

Commit 871eefe

Browse files
committed
export job
1 parent a949bd5 commit 871eefe

45 files changed

Lines changed: 3038 additions & 4 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<Project Sdk="Microsoft.NET.Sdk.Web">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net6.0</TargetFramework>
5+
<Nullable>enable</Nullable>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<EmitCompilerGeneratedFiles>true</EmitCompilerGeneratedFiles>
8+
<CompilerGeneratedFilesOutputPath>$(BaseIntermediateOutputPath)Generated</CompilerGeneratedFilesOutputPath>
9+
</PropertyGroup>
10+
11+
<ItemGroup>
12+
<PackageReference Include="Azure.Identity" />
13+
<PackageReference Include="Grpc.Net.Client" />
14+
<PackageReference Include="Microsoft.DurableTask.Generators" OutputItemType="Analyzer" />
15+
</ItemGroup>
16+
17+
<ItemGroup>
18+
<ProjectReference Include="..\..\src\Client\AzureManaged\Client.AzureManaged.csproj" />
19+
<ProjectReference Include="..\..\src\Worker\AzureManaged\Worker.AzureManaged.csproj" />
20+
<ProjectReference Include="..\..\src\ExportHistory\ExportHistory.csproj" />
21+
</ItemGroup>
22+
</Project>
23+
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
### Variables
2+
@baseUrl = http://localhost:5009
3+
@jobId = export-job-123
4+
5+
### Create a new batch export job
6+
# @name createBatchExportJob
7+
POST {{baseUrl}}/export-jobs
8+
Content-Type: application/json
9+
10+
{
11+
"jobId": "{{jobId}}",
12+
"mode": "Batch",
13+
"createdTimeFrom": "2024-01-01T00:00:00Z",
14+
"createdTimeTo": "2024-12-31T23:59:59Z",
15+
"containerName": "export-history",
16+
"prefix": "exports/",
17+
"maxInstancesPerBatch": 100,
18+
"runtimeStatus": ["Completed", "Failed"]
19+
}
20+
21+
### Create a new continuous export job
22+
# @name createContinuousExportJob
23+
POST {{baseUrl}}/export-jobs
24+
Content-Type: application/json
25+
26+
{
27+
"jobId": "export-job-continuous-123",
28+
"mode": "Continuous",
29+
"createdTimeFrom": "2024-01-01T00:00:00Z",
30+
"createdTimeTo": null,
31+
"containerName": "export-history",
32+
"prefix": "continuous-exports/",
33+
"maxInstancesPerBatch": 50
34+
}
35+
36+
### Create an export job with default storage (no container specified)
37+
# @name createExportJobWithDefaultStorage
38+
POST {{baseUrl}}/export-jobs
39+
Content-Type: application/json
40+
41+
{
42+
"jobId": "export-job-default-storage",
43+
"mode": "Batch",
44+
"createdTimeFrom": "2024-01-01T00:00:00Z",
45+
"createdTimeTo": "2024-12-31T23:59:59Z",
46+
"maxInstancesPerBatch": 100
47+
}
48+
49+
### Get a specific export job by ID
50+
# Note: This endpoint can be used to verify the export job was created and check its status
51+
# The ID in the URL should match the jobId used in create request
52+
GET {{baseUrl}}/export-jobs/{{jobId}}
53+
54+
### List all export jobs
55+
GET {{baseUrl}}/export-jobs/list
56+
57+
### List export jobs with filters
58+
# Filter by status
59+
GET {{baseUrl}}/export-jobs/list?status=Active
60+
61+
# Filter by job ID prefix
62+
GET {{baseUrl}}/export-jobs/list?jobIdPrefix=export-job-
63+
64+
# Filter by creation time range
65+
GET {{baseUrl}}/export-jobs/list?createdFrom=2024-01-01T00:00:00Z&createdTo=2024-12-31T23:59:59Z
66+
67+
# Combined filters
68+
GET {{baseUrl}}/export-jobs/list?status=Completed&jobIdPrefix=export-job-&pageSize=50
69+
70+
### Delete an export job
71+
DELETE {{baseUrl}}/export-jobs/{{jobId}}
72+
73+
### Tips:
74+
# - Replace the baseUrl variable if your application runs on a different port
75+
# - The jobId variable can be changed to test different export job instances
76+
# - Export modes:
77+
# - "Batch": Exports all instances within a time range (requires createdTimeTo)
78+
# - "Continuous": Continuously exports instances from a start time (createdTimeTo must be null)
79+
# - Runtime status filters (valid values):
80+
# - "Completed": Exports only completed orchestrations
81+
# - "Failed": Exports only failed orchestrations
82+
# - "Terminated": Exports only terminated orchestrations
83+
# - "ContinuedAsNew": Exports only continued-as-new orchestrations
84+
# - Dates are in ISO 8601 format (YYYY-MM-DDThh:mm:ssZ)
85+
# - You can use the REST Client extension in VS Code to execute these requests
86+
# - The @name directive allows referencing the response in subsequent requests
87+
# - Export jobs run asynchronously; use GET to check the status after creation
88+
Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using Microsoft.AspNetCore.Mvc;
5+
using Microsoft.DurableTask.Client;
6+
using Microsoft.DurableTask.ExportHistory;
7+
using ExportHistoryWebApp.Models;
8+
9+
namespace ExportHistoryWebApp.Controllers;
10+
11+
/// <summary>
12+
/// Controller for managing export history jobs through a REST API.
13+
/// Provides endpoints for creating, reading, listing, and deleting export jobs.
14+
/// </summary>
15+
[ApiController]
16+
[Route("export-jobs")]
17+
public class ExportJobController : ControllerBase
18+
{
19+
readonly ExportHistoryClient exportHistoryClient;
20+
readonly ILogger<ExportJobController> logger;
21+
22+
/// <summary>
23+
/// Initializes a new instance of the <see cref="ExportJobController"/> class.
24+
/// </summary>
25+
/// <param name="exportHistoryClient">Client for managing export history jobs.</param>
26+
/// <param name="logger">Logger for recording controller operations.</param>
27+
public ExportJobController(
28+
ExportHistoryClient exportHistoryClient,
29+
ILogger<ExportJobController> logger)
30+
{
31+
this.exportHistoryClient = exportHistoryClient ?? throw new ArgumentNullException(nameof(exportHistoryClient));
32+
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
33+
}
34+
35+
/// <summary>
36+
/// Creates a new export job based on the provided configuration.
37+
/// </summary>
38+
/// <param name="request">The export job creation request.</param>
39+
/// <returns>The created export job description.</returns>
40+
[HttpPost]
41+
public async Task<ActionResult<ExportJobDescription>> CreateExportJob([FromBody] CreateExportJobRequest request)
42+
{
43+
if (request == null)
44+
{
45+
return this.BadRequest("createExportJobRequest cannot be null");
46+
}
47+
48+
try
49+
{
50+
ExportDestination? destination = null;
51+
if (!string.IsNullOrEmpty(request.ContainerName))
52+
{
53+
destination = new ExportDestination(request.ContainerName)
54+
{
55+
Prefix = request.Prefix,
56+
};
57+
}
58+
59+
ExportJobCreationOptions creationOptions = new ExportJobCreationOptions(
60+
mode: request.Mode,
61+
createdTimeFrom: request.CreatedTimeFrom,
62+
createdTimeTo: request.CreatedTimeTo,
63+
destination: destination,
64+
jobId: request.JobId,
65+
format: request.Format,
66+
runtimeStatus: request.RuntimeStatus,
67+
maxInstancesPerBatch: request.MaxInstancesPerBatch);
68+
69+
ExportHistoryJobClient jobClient = await this.exportHistoryClient.CreateJobAsync(creationOptions);
70+
ExportJobDescription description = await jobClient.DescribeAsync();
71+
72+
this.logger.LogInformation("Created new export job with ID: {JobId}", description.JobId);
73+
74+
return this.CreatedAtAction(nameof(GetExportJob), new { id = description.JobId }, description);
75+
}
76+
catch (ArgumentException ex)
77+
{
78+
this.logger.LogError(ex, "Validation failed while creating export job {JobId}", request.JobId);
79+
return this.BadRequest(ex.Message);
80+
}
81+
catch (Exception ex)
82+
{
83+
this.logger.LogError(ex, "Error creating export job {JobId}", request.JobId);
84+
return this.StatusCode(500, "An error occurred while creating the export job");
85+
}
86+
}
87+
88+
/// <summary>
89+
/// Retrieves a specific export job by its ID.
90+
/// </summary>
91+
/// <param name="id">The ID of the export job to retrieve.</param>
92+
/// <returns>The export job description if found.</returns>
93+
[HttpGet("{id}")]
94+
public async Task<ActionResult<ExportJobDescription>> GetExportJob(string id)
95+
{
96+
try
97+
{
98+
ExportJobDescription? job = await this.exportHistoryClient.GetJobAsync(id);
99+
return this.Ok(job);
100+
}
101+
catch (ExportJobNotFoundException)
102+
{
103+
return this.NotFound();
104+
}
105+
catch (Exception ex)
106+
{
107+
this.logger.LogError(ex, "Error retrieving export job {JobId}", id);
108+
return this.StatusCode(500, "An error occurred while retrieving the export job");
109+
}
110+
}
111+
112+
/// <summary>
113+
/// Lists all export jobs, optionally filtered by query parameters.
114+
/// </summary>
115+
/// <param name="status">Optional filter by job status.</param>
116+
/// <param name="jobIdPrefix">Optional filter by job ID prefix.</param>
117+
/// <param name="createdFrom">Optional filter for jobs created after this time.</param>
118+
/// <param name="createdTo">Optional filter for jobs created before this time.</param>
119+
/// <param name="pageSize">Optional page size for pagination.</param>
120+
/// <param name="continuationToken">Optional continuation token for pagination.</param>
121+
/// <returns>A collection of export job descriptions.</returns>
122+
[HttpGet("list")]
123+
public async Task<ActionResult<IEnumerable<ExportJobDescription>>> ListExportJobs(
124+
[FromQuery] ExportJobStatus? status = null,
125+
[FromQuery] string? jobIdPrefix = null,
126+
[FromQuery] DateTimeOffset? createdFrom = null,
127+
[FromQuery] DateTimeOffset? createdTo = null,
128+
[FromQuery] int? pageSize = null,
129+
[FromQuery] string? continuationToken = null)
130+
{
131+
try
132+
{
133+
ExportJobQuery? query = null;
134+
if (status.HasValue || !string.IsNullOrEmpty(jobIdPrefix) || createdFrom.HasValue || createdTo.HasValue || pageSize.HasValue || !string.IsNullOrEmpty(continuationToken))
135+
{
136+
query = new ExportJobQuery
137+
{
138+
Status = status,
139+
JobIdPrefix = jobIdPrefix,
140+
CreatedFrom = createdFrom,
141+
CreatedTo = createdTo,
142+
PageSize = pageSize,
143+
ContinuationToken = continuationToken,
144+
};
145+
}
146+
147+
AsyncPageable<ExportJobDescription> jobs = this.exportHistoryClient.ListJobsAsync(query);
148+
149+
// Collect all jobs from the async pageable
150+
List<ExportJobDescription> jobList = new List<ExportJobDescription>();
151+
await foreach (ExportJobDescription job in jobs)
152+
{
153+
jobList.Add(job);
154+
}
155+
156+
return this.Ok(jobList);
157+
}
158+
catch (Exception ex)
159+
{
160+
this.logger.LogError(ex, "Error retrieving export jobs");
161+
return this.StatusCode(500, "An error occurred while retrieving export jobs");
162+
}
163+
}
164+
165+
/// <summary>
166+
/// Deletes an export job by its ID.
167+
/// </summary>
168+
/// <param name="id">The ID of the export job to delete.</param>
169+
/// <returns>No content if successful.</returns>
170+
[HttpDelete("{id}")]
171+
public async Task<IActionResult> DeleteExportJob(string id)
172+
{
173+
try
174+
{
175+
ExportHistoryJobClient jobClient = this.exportHistoryClient.GetJobClient(id);
176+
await jobClient.DeleteAsync();
177+
return this.NoContent();
178+
}
179+
catch (ExportJobNotFoundException)
180+
{
181+
return this.NotFound();
182+
}
183+
catch (Exception ex)
184+
{
185+
this.logger.LogError(ex, "Error deleting export job {JobId}", id);
186+
return this.StatusCode(500, "An error occurred while deleting the export job");
187+
}
188+
}
189+
}
190+
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
using Microsoft.DurableTask.Client;
5+
using Microsoft.DurableTask.ExportHistory;
6+
7+
namespace ExportHistoryWebApp.Models;
8+
9+
/// <summary>
10+
/// Represents a request to create a new export job.
11+
/// </summary>
12+
public class CreateExportJobRequest
13+
{
14+
/// <summary>
15+
/// Gets or sets the unique identifier for the export job. If not provided, a GUID will be generated.
16+
/// </summary>
17+
public string? JobId { get; set; }
18+
19+
/// <summary>
20+
/// Gets or sets the export mode (Batch or Continuous).
21+
/// </summary>
22+
public ExportMode Mode { get; set; }
23+
24+
/// <summary>
25+
/// Gets or sets the start time for the export (inclusive). Required.
26+
/// </summary>
27+
public DateTimeOffset CreatedTimeFrom { get; set; }
28+
29+
/// <summary>
30+
/// Gets or sets the end time for the export (inclusive). Required for Batch mode, null for Continuous mode.
31+
/// </summary>
32+
public DateTimeOffset? CreatedTimeTo { get; set; }
33+
34+
/// <summary>
35+
/// Gets or sets the blob container name where exported data will be stored. Optional if default storage is configured.
36+
/// </summary>
37+
public string? ContainerName { get; set; }
38+
39+
/// <summary>
40+
/// Gets or sets an optional prefix for blob paths.
41+
/// </summary>
42+
public string? Prefix { get; set; }
43+
44+
/// <summary>
45+
/// Gets or sets the export format settings. Optional, defaults to jsonl-gzip.
46+
/// </summary>
47+
public ExportFormat? Format { get; set; }
48+
49+
/// <summary>
50+
/// Gets or sets the orchestration runtime statuses to filter by. Optional.
51+
/// Valid statuses are: Completed, Failed, Terminated, and ContinuedAsNew.
52+
/// </summary>
53+
public List<OrchestrationRuntimeStatus>? RuntimeStatus { get; set; }
54+
55+
/// <summary>
56+
/// Gets or sets the maximum number of instances to fetch per batch. Optional, defaults to 100.
57+
/// </summary>
58+
public int? MaxInstancesPerBatch { get; set; }
59+
}
60+

0 commit comments

Comments
 (0)