konnosDashboard
DestinationPanelFields.tsx750 lines · main
1import { PermissionAction } from '@supabase/shared-types/out/constants'
2import { useParams } from 'common'
3import { Eye, EyeOff, Loader2 } from 'lucide-react'
4import { useState } from 'react'
5import type { UseFormReturn } from 'react-hook-form'
6import {
7 Button,
8 FormControl,
9 FormField,
10 Input,
11 Select,
12 SelectContent,
13 SelectGroup,
14 SelectItem,
15 SelectSeparator,
16 SelectTrigger,
17 TextArea,
18 WarningIcon,
19} from 'ui'
20import { Admonition } from 'ui-patterns'
21import { Input as PasswordInput } from 'ui-patterns/DataInputs/Input'
22import { FormItemLayout } from 'ui-patterns/form/FormItemLayout/FormItemLayout'
23
24import { CREATE_NEW_KEY, CREATE_NEW_NAMESPACE } from './DestinationForm.constants'
25import type { DestinationPanelSchemaType } from './DestinationForm.schema'
26import { InlineLink } from '@/components/ui/InlineLink'
27import { getKeys, useAPIKeysQuery } from '@/data/api-keys/api-keys-query'
28import { useAnalyticsBucketsQuery } from '@/data/storage/analytics-buckets-query'
29import { useIcebergNamespacesQuery } from '@/data/storage/iceberg-namespaces-query'
30import { useStorageCredentialsQuery } from '@/data/storage/s3-access-key-query'
31import { useAsyncCheckPermissions } from '@/hooks/misc/useCheckPermissions'
32
33export const BigQueryFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
34 return (
35 <div className="flex flex-col gap-y-6 p-5">
36 <p className="text-sm font-medium text-foreground">BigQuery settings</p>
37 <div className="flex flex-col gap-y-4">
38 <FormField
39 control={form.control}
40 name="projectId"
41 render={({ field }) => (
42 <FormItemLayout
43 layout="horizontal"
44 label="Project ID"
45 description="The Google Cloud project ID where data will be sent"
46 >
47 <FormControl>
48 <Input {...field} placeholder="my-gcp-project" />
49 </FormControl>
50 </FormItemLayout>
51 )}
52 />
53
54 <FormField
55 control={form.control}
56 name="datasetId"
57 render={({ field }) => (
58 <FormItemLayout
59 label="Dataset ID"
60 layout="horizontal"
61 description="The BigQuery dataset where replicated tables will be created"
62 >
63 <FormControl>
64 <Input {...field} placeholder="my_dataset" />
65 </FormControl>
66 </FormItemLayout>
67 )}
68 />
69
70 <FormField
71 control={form.control}
72 name="serviceAccountKey"
73 render={({ field }) => (
74 <FormItemLayout
75 layout="horizontal"
76 label="Service Account Key"
77 description="Service account credentials JSON for authenticating with BigQuery"
78 >
79 <FormControl>
80 <TextArea
81 {...field}
82 rows={5}
83 maxLength={5000}
84 placeholder='{"type": "service_account", "project_id": "...", ...}'
85 className="font-mono text-xs"
86 />
87 </FormControl>
88 </FormItemLayout>
89 )}
90 />
91 </div>
92 </div>
93 )
94}
95
96export const DuckLakeFields = ({ form }: { form: UseFormReturn<DestinationPanelSchemaType> }) => {
97 const [showCatalogUrl, setShowCatalogUrl] = useState(false)
98 const [showSecretAccessKey, setShowSecretAccessKey] = useState(false)
99
100 return (
101 <div className="flex flex-col gap-y-6 p-5">
102 <p className="text-sm font-medium text-foreground">DuckLake settings</p>
103
104 <div className="flex flex-col gap-y-1">
105 <p className="text-sm font-medium text-foreground">Catalog</p>
106 <p className="text-sm text-foreground-light">
107 Configure the PostgreSQL-backed DuckLake catalog and the S3-compatible storage location
108 for replicated data.
109 </p>
110 </div>
111
112 <div className="flex flex-col gap-y-4">
113 <FormField
114 control={form.control}
115 name="ducklakeCatalogUrl"
116 render={({ field }) => (
117 <FormItemLayout
118 layout="horizontal"
119 label="Catalog URL"
120 description="A PostgreSQL connection string for the DuckLake catalog"
121 >
122 <FormControl>
123 <PasswordInput
124 value={field.value ?? ''}
125 type={showCatalogUrl ? 'text' : 'password'}
126 placeholder="postgres://user:pass@host:5432/ducklake_catalog"
127 onChange={(event) => field.onChange(event.target.value)}
128 actions={
129 <div className="flex items-center justify-center">
130 <Button
131 type="default"
132 className="w-7"
133 icon={showCatalogUrl ? <Eye /> : <EyeOff />}
134 onClick={() => setShowCatalogUrl(!showCatalogUrl)}
135 />
136 </div>
137 }
138 />
139 </FormControl>
140 </FormItemLayout>
141 )}
142 />
143
144 <FormField
145 control={form.control}
146 name="ducklakeDataPath"
147 render={({ field }) => (
148 <FormItemLayout
149 layout="horizontal"
150 label="Data path"
151 description="An S3 path where DuckLake data files will be written"
152 >
153 <FormControl>
154 <Input {...field} placeholder="s3://bucket/path" value={field.value ?? ''} />
155 </FormControl>
156 </FormItemLayout>
157 )}
158 />
159
160 <FormField
161 control={form.control}
162 name="ducklakePoolSize"
163 render={({ field }) => (
164 <FormItemLayout
165 layout="horizontal"
166 label="Pool size"
167 description="Optional number of concurrent DuckDB connections to use"
168 >
169 <FormControl>
170 <Input
171 type="number"
172 min={1}
173 max={6}
174 value={field.value ?? ''}
175 placeholder="Default: 4"
176 onChange={(event) =>
177 field.onChange(
178 event.target.value === '' ? undefined : Number(event.target.value)
179 )
180 }
181 />
182 </FormControl>
183 </FormItemLayout>
184 )}
185 />
186 </div>
187
188 <div className="flex flex-col gap-y-1">
189 <p className="text-sm font-medium text-foreground">Object storage</p>
190 <p className="text-sm text-foreground-light">
191 Optional credentials and endpoint settings for S3-compatible storage providers.
192 </p>
193 </div>
194
195 <div className="flex flex-col gap-y-4">
196 <FormField
197 control={form.control}
198 name="ducklakeS3AccessKeyId"
199 render={({ field }) => (
200 <FormItemLayout
201 layout="horizontal"
202 label="S3 Access Key ID"
203 description="Required access key ID for the object storage provider"
204 >
205 <FormControl>
206 <Input {...field} placeholder="my-access-key" value={field.value ?? ''} />
207 </FormControl>
208 </FormItemLayout>
209 )}
210 />
211
212 <FormField
213 control={form.control}
214 name="ducklakeS3SecretAccessKey"
215 render={({ field }) => (
216 <FormItemLayout
217 layout="horizontal"
218 label="S3 Secret Access Key"
219 description="Required secret access key for the object storage provider"
220 className="relative"
221 >
222 <FormControl>
223 <Input
224 {...field}
225 type={showSecretAccessKey ? 'text' : 'password'}
226 placeholder="my-secret-key"
227 value={field.value ?? ''}
228 />
229 </FormControl>
230 <Button
231 type="default"
232 icon={showSecretAccessKey ? <Eye /> : <EyeOff />}
233 className="w-7 absolute right-6 top-[4px]"
234 onClick={() => setShowSecretAccessKey(!showSecretAccessKey)}
235 />
236 </FormItemLayout>
237 )}
238 />
239
240 <FormField
241 control={form.control}
242 name="ducklakeS3Region"
243 render={({ field }) => (
244 <FormItemLayout
245 layout="horizontal"
246 label="S3 Region"
247 description="Required region for the object storage provider"
248 >
249 <FormControl>
250 <Input {...field} placeholder="us-east-1" value={field.value ?? ''} />
251 </FormControl>
252 </FormItemLayout>
253 )}
254 />
255
256 <FormField
257 control={form.control}
258 name="ducklakeS3Endpoint"
259 render={({ field }) => (
260 <FormItemLayout
261 layout="horizontal"
262 label="S3 Endpoint"
263 description="Required endpoint without the protocol scheme, for example `127.0.0.1:5000/s3`"
264 >
265 <FormControl>
266 <Input {...field} placeholder="127.0.0.1:5000/s3" value={field.value ?? ''} />
267 </FormControl>
268 </FormItemLayout>
269 )}
270 />
271
272 <FormField
273 control={form.control}
274 name="ducklakeS3UrlStyle"
275 render={({ field }) => (
276 <FormItemLayout
277 layout="horizontal"
278 label="S3 URL style"
279 description="Choose `path` for MinIO/Briven-style endpoints or `vhost` for AWS-style virtual host addressing"
280 >
281 <FormControl>
282 <Select value={field.value ?? 'path'} onValueChange={field.onChange}>
283 <SelectTrigger>{field.value ?? 'path'}</SelectTrigger>
284 <SelectContent>
285 <SelectItem value="path">path</SelectItem>
286 <SelectItem value="vhost">vhost</SelectItem>
287 </SelectContent>
288 </Select>
289 </FormControl>
290 </FormItemLayout>
291 )}
292 />
293
294 <FormField
295 control={form.control}
296 name="ducklakeS3UseSsl"
297 render={({ field }) => (
298 <FormItemLayout
299 layout="horizontal"
300 label="Use SSL"
301 description="Whether to use SSL when connecting to the S3-compatible endpoint"
302 >
303 <FormControl>
304 <Select
305 value={field.value === false ? 'false' : 'true'}
306 onValueChange={(value) => field.onChange(value === 'true')}
307 >
308 <SelectTrigger>{field.value === false ? 'false' : 'true'}</SelectTrigger>
309 <SelectContent>
310 <SelectItem value="true">true</SelectItem>
311 <SelectItem value="false">false</SelectItem>
312 </SelectContent>
313 </Select>
314 </FormControl>
315 </FormItemLayout>
316 )}
317 />
318 </div>
319
320 <div className="flex flex-col gap-y-1">
321 <p className="text-sm font-medium text-foreground">Maintenance</p>
322 <p className="text-sm text-foreground-light">
323 Optional settings for DuckLake metadata tables and snapshot cleanup.
324 </p>
325 </div>
326
327 <div className="flex flex-col gap-y-4">
328 <FormField
329 control={form.control}
330 name="ducklakeMetadataSchema"
331 render={({ field }) => (
332 <FormItemLayout
333 layout="horizontal"
334 label="Metadata schema"
335 description="Schema used for DuckLake metadata tables in PostgreSQL"
336 >
337 <FormControl>
338 <Input {...field} placeholder="ducklake" value={field.value ?? ''} />
339 </FormControl>
340 </FormItemLayout>
341 )}
342 />
343
344 <FormField
345 control={form.control}
346 name="ducklakeExpireSnapshotsOlderThan"
347 render={({ field }) => (
348 <FormItemLayout
349 layout="horizontal"
350 label="Expire snapshots older than"
351 description="Optional snapshot retention interval, for example `7 days`"
352 >
353 <FormControl>
354 <Input {...field} placeholder="7 days" value={field.value ?? ''} />
355 </FormControl>
356 </FormItemLayout>
357 )}
358 />
359 </div>
360 </div>
361 )
362}
363
364/**
365 * [Joshen] JFYI I'd foresee a possible UX friction point here regarding S3 access key IDs and secret access keys
366 * - We'd allow users to select access key IDs via a dropdown here, but require a text input for secret access keys
367 * - Chances are most users wouldn't have the corresponding secret access key for the selected key ID at the top of their heads
368 * - So highly likely may have to default to "Create a new key" -> which from here they won't know the secret access key thereafter
369 * - And it'll end up just creating more keys for each destination
370 * Ideal scenario: Just select an access key ID, we then apply the secret access key in the PATCH request, so FE has no
371 * context of the secret access key at any point
372 */
373export const AnalyticsBucketFields = ({
374 form,
375 setIsFormInteracting,
376 onSelectNewBucket,
377}: {
378 form: UseFormReturn<DestinationPanelSchemaType>
379 setIsFormInteracting: (value: boolean) => void
380 onSelectNewBucket: () => void
381}) => {
382 const { warehouseName, s3AccessKeyId, namespace } = form.watch()
383 const [showCatalogToken, setShowCatalogToken] = useState(false)
384 const [showSecretAccessKey, setShowSecretAccessKey] = useState(false)
385
386 const { ref: projectRef } = useParams()
387
388 const { can: canReadAPIKeys } = useAsyncCheckPermissions(PermissionAction.SECRETS_READ, '*')
389 const { data: apiKeys } = useAPIKeysQuery(
390 { projectRef, reveal: true },
391 { enabled: canReadAPIKeys }
392 )
393 const { serviceKey } = getKeys(apiKeys)
394 const serviceApiKey = serviceKey?.api_key ?? ''
395
396 const {
397 data: keysData,
398 isSuccess: isSuccessKeys,
399 isPending: isLoadingKeys,
400 isError: isErrorKeys,
401 } = useStorageCredentialsQuery({ projectRef })
402 const s3Keys = keysData?.data ?? []
403 const keyNoLongerExists =
404 (s3AccessKeyId ?? '').length > 0 &&
405 s3AccessKeyId !== CREATE_NEW_KEY &&
406 !s3Keys.find((k) => k.access_key === s3AccessKeyId)
407
408 const {
409 data: analyticsBuckets = [],
410 isPending: isLoadingBuckets,
411 isError: isErrorBuckets,
412 } = useAnalyticsBucketsQuery({ projectRef })
413
414 const canSelectNamespace = !!warehouseName && !!serviceApiKey
415
416 const {
417 data: namespaces = [],
418 isPending: isLoadingNamespaces,
419 isError: isErrorNamespaces,
420 } = useIcebergNamespacesQuery(
421 { projectRef, warehouse: warehouseName },
422 { enabled: !!serviceApiKey }
423 )
424
425 return (
426 <div className="flex flex-col gap-y-6 p-5">
427 <p className="text-sm font-medium text-foreground">Analytics Bucket settings</p>
428
429 <div className="flex flex-col gap-y-4">
430 <FormField
431 control={form.control}
432 name="warehouseName"
433 render={({ field }) => (
434 <FormItemLayout
435 label="Bucket"
436 layout="horizontal"
437 description="The Analytics Bucket where data will be stored"
438 >
439 {isLoadingBuckets ? (
440 <Button
441 disabled
442 type="default"
443 className="w-full justify-between"
444 size="small"
445 iconRight={<Loader2 className="animate-spin" />}
446 >
447 Retrieving buckets
448 </Button>
449 ) : isErrorBuckets ? (
450 <Button
451 disabled
452 type="default"
453 className="w-full justify-start"
454 size="small"
455 icon={<WarningIcon />}
456 >
457 Failed to retrieve buckets
458 </Button>
459 ) : (
460 <FormControl>
461 <Select
462 value={field.value}
463 onValueChange={(value) => {
464 if (value === 'new-bucket') {
465 onSelectNewBucket()
466 } else {
467 setIsFormInteracting(true)
468 field.onChange(value)
469 // [Joshen] Ideally should select the first namespace of the selected bucket
470 form.setValue('namespace', '')
471 }
472 }}
473 >
474 <SelectTrigger>{field.value || 'Select a bucket'}</SelectTrigger>
475 <SelectContent>
476 <SelectGroup>
477 {analyticsBuckets.length === 0 ? (
478 <SelectItem value="__no_buckets__" disabled>
479 No buckets available
480 </SelectItem>
481 ) : (
482 analyticsBuckets.map((bucket) => (
483 <SelectItem key={bucket.name} value={bucket.name}>
484 {bucket.name}
485 </SelectItem>
486 ))
487 )}
488 <SelectSeparator />
489 <SelectItem value="new-bucket">Create a new bucket</SelectItem>
490 </SelectGroup>
491 </SelectContent>
492 </Select>
493 </FormControl>
494 )}
495 </FormItemLayout>
496 )}
497 />
498
499 <FormField
500 control={form.control}
501 name="namespace"
502 render={({ field }) => (
503 <FormItemLayout
504 label="Namespace"
505 layout="horizontal"
506 description="The namespace within the bucket where tables will be organized"
507 >
508 {isLoadingNamespaces && canSelectNamespace ? (
509 <Button
510 disabled
511 type="default"
512 className="w-full justify-between"
513 size="small"
514 iconRight={<Loader2 className="animate-spin" />}
515 >
516 Retrieving namespaces
517 </Button>
518 ) : isErrorNamespaces ? (
519 <Button
520 disabled
521 type="default"
522 className="w-full justify-start"
523 size="small"
524 icon={<WarningIcon />}
525 >
526 Failed to retrieve namespaces
527 </Button>
528 ) : (
529 <FormControl>
530 <Select
531 value={field.value}
532 onValueChange={(value) => {
533 setIsFormInteracting(true)
534 field.onChange(value)
535 }}
536 disabled={!canSelectNamespace}
537 >
538 <SelectTrigger>
539 {!canSelectNamespace
540 ? 'Select a warehouse first'
541 : field.value === CREATE_NEW_NAMESPACE
542 ? 'Create a new namespace'
543 : field.value || 'Select a namespace'}
544 </SelectTrigger>
545 <SelectContent>
546 <SelectGroup>
547 {namespaces.length === 0 ? (
548 <SelectItem value="__no_namespaces__" disabled>
549 No namespaces available
550 </SelectItem>
551 ) : (
552 namespaces.map((namespace) => (
553 <SelectItem key={namespace} value={namespace}>
554 {namespace}
555 </SelectItem>
556 ))
557 )}
558 <SelectSeparator />
559 <SelectItem key={CREATE_NEW_NAMESPACE} value={CREATE_NEW_NAMESPACE}>
560 Create a new namespace
561 </SelectItem>
562 </SelectGroup>
563 </SelectContent>
564 </Select>
565 </FormControl>
566 )}
567 </FormItemLayout>
568 )}
569 />
570
571 {namespace === CREATE_NEW_NAMESPACE && (
572 <FormField
573 control={form.control}
574 name="newNamespaceName"
575 render={({ field }) => (
576 <FormItemLayout
577 label="New Namespace Name"
578 layout="horizontal"
579 description="A unique name for the new namespace"
580 >
581 <FormControl>
582 <Input {...field} placeholder="new_namespace" value={field.value || ''} />
583 </FormControl>
584 </FormItemLayout>
585 )}
586 />
587 )}
588
589 <FormField
590 control={form.control}
591 name="catalogToken"
592 render={({ field }) => (
593 <FormItemLayout
594 layout="horizontal"
595 label="Catalog Token"
596 description={
597 <>
598 Automatically retrieved from your project's{' '}
599 <InlineLink href={`/project/${projectRef}/settings/api-keys`}>
600 service role key
601 </InlineLink>
602 </>
603 }
604 >
605 <PasswordInput
606 disabled
607 value={field.value}
608 type={showCatalogToken ? 'text' : 'password'}
609 placeholder="Auto-populated"
610 actions={
611 serviceApiKey ? (
612 <div className="flex items-center justify-center">
613 <Button
614 type="default"
615 className="w-7"
616 icon={showCatalogToken ? <Eye /> : <EyeOff />}
617 onClick={() => setShowCatalogToken(!showCatalogToken)}
618 />
619 </div>
620 ) : null
621 }
622 />
623 </FormItemLayout>
624 )}
625 />
626
627 <FormField
628 control={form.control}
629 name="s3AccessKeyId"
630 render={({ field }) => (
631 <FormItemLayout
632 layout="horizontal"
633 label="S3 Access Key ID"
634 description={
635 <div className="flex flex-col gap-y-2">
636 <p>
637 Access keys are managed in your Storage{' '}
638 <InlineLink href={`/project/${projectRef}/storage/s3`}>S3 settings</InlineLink>
639 </p>
640
641 {isSuccessKeys && keyNoLongerExists && (
642 <Admonition
643 type="warning"
644 title="Unable to find access key ID in project"
645 description={
646 <>
647 Please select another key or create a new set, as this destination will
648 not work otherwise. S3 access keys can be managed in your{' '}
649 <InlineLink href={`/project/${projectRef}/storage/files/settings`}>
650 storage settings
651 </InlineLink>
652 .
653 </>
654 }
655 />
656 )}
657
658 {s3AccessKeyId === CREATE_NEW_KEY && (
659 <Admonition
660 type="default"
661 description="A new set of S3 access keys will be created."
662 />
663 )}
664 </div>
665 }
666 >
667 {isLoadingKeys ? (
668 <Button
669 disabled
670 type="default"
671 className="w-full justify-between"
672 size="small"
673 iconRight={<Loader2 className="animate-spin" />}
674 >
675 Retrieving keys
676 </Button>
677 ) : isErrorKeys ? (
678 <Button
679 disabled
680 type="default"
681 className="w-full justify-start"
682 size="small"
683 icon={<WarningIcon />}
684 >
685 Failed to retrieve keys
686 </Button>
687 ) : (
688 <FormControl>
689 <Select value={field.value} onValueChange={field.onChange}>
690 <SelectTrigger>
691 {field.value === CREATE_NEW_KEY
692 ? 'Create a new key'
693 : (field.value ?? '').length === 0
694 ? 'Select an access key ID'
695 : field.value}
696 </SelectTrigger>
697 <SelectContent>
698 <SelectGroup>
699 {s3Keys.map((key) => (
700 <SelectItem key={key.id} value={key.access_key}>
701 {key.access_key}
702 <p className="text-foreground-lighter">{key.description}</p>
703 </SelectItem>
704 ))}
705 <SelectSeparator />
706 <SelectItem key={CREATE_NEW_KEY} value={CREATE_NEW_KEY}>
707 Create a new key
708 </SelectItem>
709 </SelectGroup>
710 </SelectContent>
711 </Select>
712 </FormControl>
713 )}
714 </FormItemLayout>
715 )}
716 />
717
718 {s3AccessKeyId !== CREATE_NEW_KEY && (
719 <FormField
720 control={form.control}
721 name="s3SecretAccessKey"
722 render={({ field }) => (
723 <FormItemLayout
724 layout="horizontal"
725 label="S3 Secret Access Key"
726 className="relative"
727 description="The secret key corresponding to your selected access key ID."
728 >
729 <FormControl>
730 <Input
731 {...field}
732 type={showSecretAccessKey ? 'text' : 'password'}
733 value={field.value ?? ''}
734 placeholder="Provide the secret access key"
735 />
736 </FormControl>
737 <Button
738 type="default"
739 icon={showSecretAccessKey ? <Eye /> : <EyeOff />}
740 className="w-7 absolute right-6 top-[4px]"
741 onClick={() => setShowSecretAccessKey(!showSecretAccessKey)}
742 />
743 </FormItemLayout>
744 )}
745 />
746 )}
747 </div>
748 </div>
749 )
750}