feat: wire CSV segments to the cohorts API (#8295)

Author: Zaimwa9

frontend/common/services/useCohort.ts

@@ -0,0 +1,82 @@
+import { Res } from 'common/types/responses'
+import { Req } from 'common/types/requests'
+import { service } from 'common/service'
+import toFormData from 'common/utils/toFormData'
+
+export const cohortService = service
+  .enhanceEndpoints({ addTagTypes: ['Cohort', 'Segment'] })
+  .injectEndpoints({
+    endpoints: (builder) => ({
+      createCohort: builder.mutation<Res['cohort'], Req['createCohort']>({
+        invalidatesTags: (q, e, arg) => [
+          { id: 'LIST', type: 'Cohort' },
+          { id: `LIST${arg.projectId}`, type: 'Segment' },
+        ],
+        query: (query) => ({
+          body: {
+            description: query.description,
+            metadata: query.metadata,
+            name: query.name,
+          },
+          method: 'POST',
+          url: `environments/${query.environmentApiKey}/cohorts/`,
+        }),
+      }),
+      deleteCohort: builder.mutation<void, Req['deleteCohort']>({
+        invalidatesTags: (q, e, arg) => [
+          { id: 'LIST', type: 'Cohort' },
+          { id: `LIST${arg.projectId}`, type: 'Segment' },
+        ],
+        query: (query) => ({
+          method: 'DELETE',
+          url: `environments/${query.environmentApiKey}/cohorts/${query.cohortId}/`,
+        }),
+      }),
+      syncCohortCsv: builder.mutation<
+        Res['cohortCsvSync'],
+        Req['syncCohortCsv']
+      >({
+        invalidatesTags: (q, e, arg) => [
+          { id: arg.cohortId, type: 'Cohort' },
+          { id: `LIST${arg.projectId}`, type: 'Segment' },
+        ],
+        queryFn: async (query, baseQueryApi, extraOptions, baseQuery) => {
+          // projectId only feeds tag invalidation; keep it out of the form data.
+          const { cohortId, environmentApiKey, projectId: _, ...rest } = query
+          const formData = toFormData({ ...rest })
+          const { data, error } = await baseQuery({
+            body: formData,
+            method: 'POST',
+            url: `environments/${environmentApiKey}/cohorts/${cohortId}/sync-csv/`,
+          })
+          return { data, error } as {
+            data: Res['cohortCsvSync']
+            error: never
+          }
+        },
+      }),
+      // END OF ENDPOINTS
+    }),
+  })
+
+export async function deleteCohort(
+  store: any,
+  data: Req['deleteCohort'],
+  options?: Parameters<typeof cohortService.endpoints.deleteCohort.initiate>[1],
+) {
+  return store.dispatch(
+    cohortService.endpoints.deleteCohort.initiate(data, options),
+  )
+}
+
+export const {
+  useCreateCohortMutation,
+  useDeleteCohortMutation,
+  useSyncCohortCsvMutation,
+  // END OF EXPORTS
+} = cohortService
+
+/* Usage examples:
+const [createCohort, { isLoading, data, isSuccess }] = useCreateCohortMutation()
+const [syncCohortCsv, { isLoading }] = useSyncCohortCsvMutation()
+*/

frontend/common/types/requests.ts

@@ -5,6 +5,7 @@ import {
   FeatureStateValue,
   ImportStrategy,
   Approval,
+  Metadata,
   MultivariateOption,
   SAMLConfiguration,
   Segment,
@@ -173,6 +174,26 @@ export type Req = {
     projectId: number
     segment: Omit<Segment, 'id' | 'uuid' | 'project'>
   }
+  createCohort: {
+    environmentApiKey: string
+    projectId: number
+    name: string
+    description?: string
+    metadata?: Metadata[]
+  }
+  deleteCohort: {
+    environmentApiKey: string
+    cohortId: number
+    projectId: number
+  }
+  syncCohortCsv: {
+    environmentApiKey: string
+    cohortId: number
+    projectId: number
+    file: File
+    identifier_column?: number
+    has_header?: boolean
+  }
   cloneSegment: {
     projectId: number
     segmentId: number

frontend/common/types/responses.ts

@@ -174,6 +174,16 @@ export type SegmentMembersResponse = PagedResponse<SegmentMember> & {
   // Pass as `cursor` to fetch the next page; null when there are no more rows.
   next_cursor: string | null
 }
+export type SegmentCohort = {
+  id: number
+  environment: number
+  environment_api_key: string
+  environment_name: string
+  source_type: 'csv'
+  version: number
+  deletion_requested_at: string | null
+}
+
 export type Segment = {
   id: number
   rules: SegmentRule[]
@@ -184,6 +194,7 @@ export type Segment = {
   feature?: number
   metadata: Metadata[] | []
   membership_counts?: SegmentMembership[]
+  cohort?: SegmentCohort | null
 }
 export type ProjectChangeRequest = Omit<
   ChangeRequest,
@@ -995,6 +1006,29 @@ export type Metadata = {
   field_value: string
 }
 
+export type Cohort = {
+  id: number
+  uuid: string
+  name: string
+  description: string | null
+  segment: number
+  source_type: 'csv'
+  version: number
+  created_at: string
+}
+
+export type CohortCsvSyncResult = {
+  version: number
+  added: number
+  removed: number
+  unchanged: number
+  ignored: {
+    empty: number
+    duplicates: number
+    too_long: number
+  }
+}
+
 export type MetadataFieldModelField = {
   id: number
   content_type: number
@@ -1324,6 +1358,8 @@ export type TrustRelationship = {
 export type Res = {
   segments: PagedResponse<Segment>
   segment: Segment
+  cohort: Cohort
+  cohortCsvSync: CohortCsvSyncResult
   segmentMembers: SegmentMembersResponse
   auditLogs: PagedResponse<AuditLogItem>
   organisationLicence: {}

frontend/common/utils/__tests__/csv.test.ts

@@ -1,4 +1,10 @@
-import { extractIdentifiers, parseCsvText, toParsedCsv } from 'common/utils/csv'
+import {
+  extractIdentifiers,
+  MAX_IDENTIFIER_BYTES,
+  parseCsvText,
+  toCsvColumn,
+  toParsedCsv,
+} from 'common/utils/csv'
 
 describe('parseCsvText', () => {
   const cases: [string, string, string[][]][] = [
@@ -68,6 +74,7 @@ describe('extractIdentifiers', () => {
       duplicateCount: 2,
       emptyCount: 2,
       identifiers: ['a', 'b'],
+      tooLongCount: 0,
     })
   })
 
@@ -76,6 +83,40 @@ describe('extractIdentifiers', () => {
       duplicateCount: 0,
       emptyCount: 1,
       identifiers: ['y'],
+      tooLongCount: 0,
     })
   })
+
+  test('identifiers over the UTF-8 byte limit are dropped', () => {
+    // 'é' is 2 UTF-8 bytes, so 513 of them exceed 1024 bytes in 513 chars.
+    const rows = [
+      ['a'.repeat(MAX_IDENTIFIER_BYTES)],
+      ['a'.repeat(MAX_IDENTIFIER_BYTES + 1)],
+      ['é'.repeat(513)],
+    ]
+    expect(extractIdentifiers(rows, 0)).toEqual({
+      duplicateCount: 0,
+      emptyCount: 0,
+      identifiers: ['a'.repeat(MAX_IDENTIFIER_BYTES)],
+      tooLongCount: 2,
+    })
+  })
+})
+
+describe('toCsvColumn', () => {
+  test.each([
+    ['plain values', ['a', 'b'], 'a\nb'],
+    ['comma quoted', ['Doe, Jane', 'b'], '"Doe, Jane"\nb'],
+    ['quote escaped', ['say "hi"'], '"say ""hi"""'],
+    ['newline quoted', ['line1\nline2'], '"line1\nline2"'],
+  ])('%s', (_, values, expected) => {
+    expect(toCsvColumn(values)).toEqual(expected)
+  })
+
+  test('round-trips through parseCsvText', () => {
+    const values = ['plain', 'Doe, Jane', 'say "hi"', 'multi\nline']
+    expect(parseCsvText(toCsvColumn(values)).map((row) => row[0])).toEqual(
+      values,
+    )
+  })
 })

frontend/common/utils/csv.ts

@@ -7,8 +7,13 @@ export type ExtractedIdentifiers = {
   duplicateCount: number
   emptyCount: number
   identifiers: string[]
+  tooLongCount: number
 }
 
+// Mirrors the API's COHORT_IDENTIFIER_MAX_BYTES (Edge identifiers are
+// DynamoDB sort keys, capped at 1024 bytes).
+export const MAX_IDENTIFIER_BYTES = 1024
+
 export function parseCsvText(text: string): string[][] {
   const rows: string[][] = []
   let row: string[] = []
@@ -58,7 +63,10 @@ export function toParsedCsv(
   if (!rawRows.length) {
     return { columns: [], rows: [] }
   }
-  const columnCount = Math.max(...rawRows.map((cells) => cells.length))
+  const columnCount = rawRows.reduce(
+    (max, cells) => Math.max(max, cells.length),
+    0,
+  )
   if (hasHeaders) {
     const [header, ...rows] = rawRows
     return {
@@ -81,18 +89,30 @@ export function extractIdentifiers(
 ): ExtractedIdentifiers {
   const seen = new Set<string>()
   const identifiers: string[] = []
+  const encoder = new TextEncoder()
   let emptyCount = 0
   let duplicateCount = 0
+  let tooLongCount = 0
   for (const cells of rows) {
     const value = (cells[columnIndex] ?? '').trim()
     if (!value) {
       emptyCount++
+    } else if (encoder.encode(value).length > MAX_IDENTIFIER_BYTES) {
+      tooLongCount++
     } else if (seen.has(value)) {
       duplicateCount++
     } else {
       seen.add(value)
       identifiers.push(value)
     }
   }
-  return { duplicateCount, emptyCount, identifiers }
+  return { duplicateCount, emptyCount, identifiers, tooLongCount }
+}
+
+export function toCsvColumn(values: string[]): string {
+  return values
+    .map((value) =>
+      /[",\r\n]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value,
+    )
+    .join('\n')
 }

frontend/web/components/CsvUpload/CsvUpload.tsx

@@ -9,6 +9,7 @@ import './CsvUpload.scss'
 
 export type CsvUploadType = {
   value: File | null
+  maxSizeBytes?: number
   rowCount?: number
   onChange: (file: File, text: string) => void
 }
@@ -20,7 +21,12 @@ const formatFileSize = (bytes: number) => {
   return `${(bytes / 1024).toFixed(1)} KB`
 }
 
-const CsvUpload: FC<CsvUploadType> = ({ onChange, rowCount, value }) => {
+const CsvUpload: FC<CsvUploadType> = ({
+  maxSizeBytes,
+  onChange,
+  rowCount,
+  value,
+}) => {
   const [error, setError] = useState('')
 
   const onDrop = useCallback(
@@ -46,12 +52,17 @@ const CsvUpload: FC<CsvUploadType> = ({ onChange, rowCount, value }) => {
     accept: {
       'text/csv': ['.csv'],
     },
+    maxSize: maxSizeBytes,
     multiple: false,
     noClick: true,
     noKeyboard: true,
     onDrop,
-    onDropRejected: () => {
-      setError('Please select a CSV file')
+    onDropRejected: (rejections) => {
+      setError(
+        rejections[0]?.errors?.[0]?.code === 'file-too-large' && maxSizeBytes
+          ? `Please select a file smaller than ${formatFileSize(maxSizeBytes)}`
+          : 'Please select a CSV file',
+      )
     },
   })
 
@@ -91,7 +102,11 @@ const CsvUpload: FC<CsvUploadType> = ({ onChange, rowCount, value }) => {
           </div>
         )}
       </div>
-      {!!error && <ErrorMessage error={error} />}
+      {!!error && (
+        <div className='mt-3'>
+          <ErrorMessage error={error} />
+        </div>
+      )}
     </div>
   )
 }

frontend/web/components/modals/ConfirmRemoveSegment.tsx

@@ -6,6 +6,7 @@ import Utils from 'common/utils/utils'
 import Button from 'components/base/forms/Button'
 import ModalHR from './ModalHR'
 import { deleteSegment } from 'common/services/useSegment'
+import { deleteCohort } from 'common/services/useCohort'
 import { getStore } from 'common/store'
 
 type ConfirmRemoveSegmentType = {
@@ -19,7 +20,15 @@ export const handleRemoveSegment = (
 ) => {
   const removeSegmentCallback = async () => {
     try {
-      const res = await deleteSegment(getStore(), { id: segment.id, projectId })
+      // Cohort-managed segments must be deleted via their cohort; the segment
+      // endpoint rejects them.
+      const res = segment.cohort
+        ? await deleteCohort(getStore(), {
+            cohortId: segment.cohort.id,
+            environmentApiKey: segment.cohort.environment_api_key,
+            projectId: Number(projectId),
+          })
+        : await deleteSegment(getStore(), { id: segment.id, projectId })
       if (res.error) throw new Error(res.error)
       toast(
         <div>