konnosDashboard
storage-explorer.tsx1912 lines · main
1import { BlobReader, BlobWriter, ZipWriter } from '@zip.js/zip.js'
2import { IS_PLATFORM } from 'common'
3import { capitalize, chunk, compact, find, findIndex, has, isObject, uniq, uniqBy } from 'lodash'
4import { createContext, PropsWithChildren, useContext, useEffect, useState } from 'react'
5import { useLatest } from 'react-use'
6import { toast } from 'sonner'
7import * as tus from 'tus-js-client'
8import { Button, SONNER_DEFAULT_DURATION, SonnerProgress } from 'ui'
9import { proxy, useSnapshot } from 'valtio'
10
11import { useSelectedBucket } from '@/components/interfaces/Storage/FilesBuckets/useSelectedBucket'
12import {
13 STORAGE_ROW_STATUS,
14 STORAGE_ROW_TYPES,
15} from '@/components/interfaces/Storage/Storage.constants'
16import {
17 StorageColumn,
18 StorageItem,
19 StorageItemMetadata,
20 StorageItemWithColumn,
21} from '@/components/interfaces/Storage/Storage.types'
22import {
23 calculateTotalRemainingTime,
24 EMPTY_FOLDER_PLACEHOLDER_FILE_NAME,
25 formatFolderItems,
26 formatTime,
27 getFilesDataTransferItems,
28 getPathAlongFoldersToIndex,
29 sanitizeNameForDuplicateInColumn,
30 validateFolderName,
31} from '@/components/interfaces/Storage/StorageExplorer/StorageExplorer.utils'
32import { fetchFileUrl } from '@/components/interfaces/Storage/StorageExplorer/useFetchFileUrlQuery'
33import { getStoragePreference } from '@/components/interfaces/Storage/StorageExplorer/useStoragePreference'
34import { convertFromBytes } from '@/components/interfaces/Storage/StorageSettings/StorageSettings.utils'
35import { InlineLink } from '@/components/ui/InlineLink'
36import { getOrRefreshTemporaryApiKey } from '@/data/api-keys/temp-api-keys-utils'
37import { configKeys } from '@/data/config/keys'
38import { useProjectApiUrl } from '@/data/config/project-endpoint-query'
39import type { ProjectStorageConfigResponse } from '@/data/config/project-storage-config-query'
40import { getQueryClient } from '@/data/query-client'
41import { deleteBucketObject } from '@/data/storage/bucket-object-delete-mutation'
42import { signBucketObjects } from '@/data/storage/bucket-object-sign-mutation'
43import { listBucketObjects, StorageObject } from '@/data/storage/bucket-objects-list-mutation'
44import { deleteBucketPrefix } from '@/data/storage/bucket-prefix-delete-mutation'
45import type { Bucket } from '@/data/storage/buckets-query'
46import { moveStorageObject } from '@/data/storage/object-move-mutation'
47import { useSelectedProjectQuery } from '@/hooks/misc/useSelectedProject'
48import { PROJECT_STATUS } from '@/lib/constants'
49import { lookupMime } from '@/lib/mime'
50import { createProjectBrivenClient } from '@/lib/project-briven-client'
51import { ResponseError } from '@/types'
52
53type UploadProgress = {
54 percentage: number
55 elapsed: number
56 uploadSpeed: number
57 remainingBytes: number
58 remainingTime: number
59}
60
61const LIMIT = 200
62const OFFSET = 0
63const DEFAULT_RETRY_SECONDS = 5
64const RATE_LIMIT_RETRY_SECONDS = 60
65
66const STORAGE_PROGRESS_INFO_TEXT = "Do not close the browser until it's completed"
67
68let abortController: AbortController
69if (typeof window !== 'undefined') {
70 abortController = new AbortController()
71}
72
73function createStorageExplorerState({
74 projectRef,
75 connectionString,
76 bucket,
77 resumableUploadUrl,
78 clientEndpoint,
79}: {
80 projectRef: string
81 connectionString: string
82 bucket?: Bucket
83 resumableUploadUrl: string
84 clientEndpoint: string
85}) {
86 const getSortOptions = () => {
87 const { sortBy, sortByOrder } = getStoragePreference(projectRef)
88 return { column: sortBy, order: sortByOrder }
89 }
90
91 const state = proxy({
92 projectRef,
93 connectionString,
94 resumableUploadUrl,
95 uploadProgresses: [] as UploadProgress[],
96 selectedBucket: bucket as Bucket,
97
98 abortApiCalls: () => {
99 if (abortController) {
100 abortController.abort()
101 abortController = new AbortController()
102 }
103 },
104
105 abortUploadCallbacks: {} as { [key: string]: (() => void)[] },
106 abortUploads: (toastId: string | number) => {
107 state.abortUploadCallbacks[toastId].forEach((callback) => callback())
108 state.abortUploadCallbacks[toastId] = []
109 },
110
111 columns: [] as StorageColumn[],
112 popColumn: () => {
113 state.abortApiCalls()
114 state.columns = state.columns.slice(0, state.getLatestColumnIndex())
115 },
116 popColumnAtIndex: (index: number) => {
117 state.columns = state.columns.slice(0, index + 1)
118 },
119 pushColumnAtIndex: (column: StorageColumn, index: number) => {
120 state.columns = state.columns.slice(0, index + 1).concat([column])
121 },
122
123 openedFolders: [] as StorageItem[],
124 pushOpenedFolderAtIndex: (folder: StorageItem, index: number) => {
125 state.openedFolders = state.openedFolders.slice(0, index).concat(folder)
126 },
127 popOpenedFolders: () => {
128 state.openedFolders = state.openedFolders.slice(0, state.openedFolders.length - 1)
129 },
130 popOpenedFoldersAtIndex: (index: number) => {
131 state.openedFolders = state.openedFolders.slice(0, index + 1)
132 },
133 clearOpenedFolders: () => {
134 state.openedFolders = []
135 },
136
137 selectedItems: [] as StorageItemWithColumn[],
138 setSelectedItems: (items: StorageItemWithColumn[]) => (state.selectedItems = items),
139 clearSelectedItems: (columnIndex?: number) => {
140 if (columnIndex !== undefined) {
141 state.selectedItems = state.selectedItems.filter((item) => item.columnIndex !== columnIndex)
142 } else {
143 state.selectedItems = []
144 }
145 },
146
147 selectedItemsToDelete: [] as StorageItemWithColumn[],
148 setSelectedItemsToDelete: (items: StorageItemWithColumn[]) => {
149 state.selectedItemsToDelete = items
150 },
151
152 selectedItemsToMove: [] as StorageItemWithColumn[],
153 setSelectedItemsToMove: (items: StorageItemWithColumn[]) => {
154 state.selectedItemsToMove = items
155 },
156
157 setSelectedItemToRename: (file: { name: string; columnIndex: number }) => {
158 state.updateRowStatus({
159 name: file.name,
160 status: STORAGE_ROW_STATUS.EDITING,
161 columnIndex: file.columnIndex,
162 })
163 },
164
165 isSearching: false,
166 setIsSearching: (value: boolean) => (state.isSearching = value),
167
168 isRefreshing: false,
169
170 selectedFilePreview: undefined as StorageItemWithColumn | undefined,
171 setSelectedFilePreview: (file?: StorageItemWithColumn) => (state.selectedFilePreview = file),
172
173 selectedFileCustomExpiry: undefined as StorageItem | undefined,
174 setSelectedFileCustomExpiry: (item?: StorageItem) => (state.selectedFileCustomExpiry = item),
175
176 // Functions that manage the UI of the Storage Explorer
177
178 getLatestColumnIndex: () => {
179 return state.columns.length - 1
180 },
181
182 setColumnIsLoadingMore: (index: number, isLoadingMoreItems: boolean = true) => {
183 state.columns = state.columns.map((col, idx) => {
184 return idx === index ? { ...col, isLoadingMoreItems } : col
185 })
186 },
187
188 // ======== Folders CRUD ========
189
190 addNewFolderPlaceholder: (columnIndex: number) => {
191 const isPrepend = true
192 const folderName = 'Untitled folder'
193 const folderType = STORAGE_ROW_TYPES.FOLDER
194 const columnIdx = columnIndex === -1 ? state.getLatestColumnIndex() : columnIndex
195 state.addTempRow({
196 type: folderType,
197 name: folderName,
198 status: STORAGE_ROW_STATUS.EDITING,
199 columnIndex: columnIdx,
200 metadata: null,
201 isPrepend,
202 })
203 },
204
205 addNewFolder: async ({
206 folderName,
207 columnIndex,
208 onError,
209 }: {
210 folderName: string
211 columnIndex: number
212 onError?: () => void
213 }) => {
214 const autofix = false
215 const formattedName = sanitizeNameForDuplicateInColumn(state, {
216 name: folderName,
217 autofix,
218 columnIndex,
219 })
220
221 if (formattedName === null) {
222 onError?.()
223 return
224 }
225
226 if (formattedName.length === 0) {
227 return state.removeTempRows(columnIndex)
228 }
229
230 const folderNameError = validateFolderName(formattedName)
231 if (folderNameError) {
232 onError?.()
233 return toast.error(folderNameError)
234 }
235
236 state.updateFolderAfterEdit({ folderName: formattedName, columnIndex })
237
238 const emptyPlaceholderFile = `${formattedName}/${EMPTY_FOLDER_PLACEHOLDER_FILE_NAME}`
239 const pathToFolder = state.openedFolders
240 .slice(0, columnIndex)
241 .map((folder) => folder.name)
242 .join('/')
243 const formattedPathToEmptyPlaceholderFile =
244 pathToFolder.length > 0 ? `${pathToFolder}/${emptyPlaceholderFile}` : emptyPlaceholderFile
245
246 const client = await createProjectBrivenClient(state.projectRef, clientEndpoint)
247 await client.storage
248 .from(state.selectedBucket.name)
249 .upload(
250 formattedPathToEmptyPlaceholderFile,
251 new File([], EMPTY_FOLDER_PLACEHOLDER_FILE_NAME)
252 )
253
254 if (pathToFolder.length > 0) {
255 await deleteBucketObject({
256 projectRef: state.projectRef,
257 bucketId: state.selectedBucket.id,
258 paths: [`${pathToFolder}/${EMPTY_FOLDER_PLACEHOLDER_FILE_NAME}`],
259 })
260 }
261
262 const newFolder = state.columns[columnIndex]?.items?.find((x) => x?.name === formattedName)
263 if (newFolder) state.openFolder(columnIndex, newFolder)
264 },
265
266 fetchFolderContents: async ({
267 bucketId,
268 folderId,
269 folderName,
270 index,
271 searchString,
272 }: {
273 bucketId: string
274 folderId: string | null
275 folderName: string
276 index: number
277 searchString?: string
278 }) => {
279 state.abortApiCalls()
280 state.updateRowStatus({
281 name: folderName,
282 status: STORAGE_ROW_STATUS.LOADING,
283 columnIndex: index,
284 })
285 const prefix = state.openedFolders
286 .slice(0, index + 1)
287 .map((folder) => folder.name)
288 .join('/')
289 state.pushColumnAtIndex(
290 {
291 id: folderId,
292 name: folderName,
293 path: prefix,
294 status: STORAGE_ROW_STATUS.LOADING,
295 items: [],
296 },
297 index
298 )
299
300 const options = {
301 limit: LIMIT,
302 offset: OFFSET,
303 search: searchString,
304 sortBy: getSortOptions(),
305 }
306
307 try {
308 const data = await listBucketObjects(
309 {
310 bucketId,
311 projectRef: state.projectRef,
312 path: prefix,
313 options,
314 },
315 abortController?.signal
316 )
317
318 state.updateRowStatus({
319 name: folderName,
320 status: STORAGE_ROW_STATUS.READY,
321 columnIndex: index,
322 })
323 const formattedItems = formatFolderItems(data, prefix)
324 state.pushColumnAtIndex(
325 {
326 id: folderId || folderName,
327 name: folderName,
328 path: prefix,
329 status: STORAGE_ROW_STATUS.READY,
330 items: formattedItems,
331 hasMoreItems: formattedItems.length === LIMIT,
332 isLoadingMoreItems: false,
333 },
334 index
335 )
336 } catch (error: any) {
337 if (error.name === 'AbortError') {
338 state.updateRowStatus({
339 name: folderName,
340 status: STORAGE_ROW_STATUS.READY,
341 columnIndex: index,
342 })
343 } else {
344 toast.error(`Failed to retrieve folder contents from "${folderName}": ${error.message}`)
345 }
346 }
347 },
348
349 fetchMoreFolderContents: async ({
350 index,
351 column,
352 searchString = '',
353 }: {
354 index: number
355 column: StorageColumn
356 searchString?: string
357 }) => {
358 state.setColumnIsLoadingMore(index)
359
360 const options = {
361 limit: LIMIT,
362 offset: column.items.length,
363 search: searchString,
364 sortBy: getSortOptions(),
365 }
366
367 try {
368 const data = await listBucketObjects(
369 {
370 projectRef: state.projectRef,
371 bucketId: state.selectedBucket.id,
372 path: column.path,
373 options,
374 },
375 abortController?.signal
376 )
377
378 // Add items to column
379 const formattedItems = formatFolderItems(data, column.path)
380 state.columns = state.columns.map((col, idx) => {
381 if (idx === index) {
382 return {
383 ...col,
384 items: col.items.concat(formattedItems),
385 isLoadingMoreItems: false,
386 hasMoreItems: data.length === LIMIT,
387 }
388 }
389 return col
390 })
391 } catch (error: any) {
392 if (!error.message.includes('aborted')) {
393 toast.error(`Failed to retrieve more folder contents: ${error.message}`)
394 }
395 }
396 },
397
398 refetchAllOpenedFolders: async () => {
399 const paths = state.openedFolders.map((folder) => folder.name)
400 await state.fetchFoldersByPath({ paths })
401 },
402
403 refreshAll: async () => {
404 state.isRefreshing = true
405 try {
406 await state.refetchAllOpenedFolders()
407 } finally {
408 state.isRefreshing = false
409 }
410 },
411
412 fetchFoldersByPath: async ({
413 paths,
414 searchString = '',
415 showLoading = false,
416 }: {
417 paths: string[]
418 searchString?: string
419 showLoading?: boolean
420 }) => {
421 if (state.selectedBucket.id === undefined) return
422
423 const pathsWithEmptyPrefix = [''].concat(paths)
424
425 if (showLoading) {
426 state.columns = [state.selectedBucket.name].concat(paths).map((path) => {
427 return { id: path, name: path, path, status: STORAGE_ROW_STATUS.LOADING, items: [] }
428 })
429 }
430
431 const foldersItems = await Promise.all(
432 pathsWithEmptyPrefix.map(async (_path, idx) => {
433 const prefix = paths.slice(0, idx).join('/')
434 const options = {
435 limit: LIMIT,
436 offset: OFFSET,
437 search: searchString,
438 sortBy: getSortOptions(),
439 }
440
441 try {
442 const data = await listBucketObjects({
443 projectRef: state.projectRef,
444 bucketId: state.selectedBucket.id,
445 path: prefix,
446 options,
447 })
448 return data
449 } catch (error: any) {
450 toast.error(`Failed to fetch folders: ${error.message}`)
451 return []
452 }
453 })
454 )
455
456 const formattedFolders = foldersItems.map((folderItems, idx) => {
457 const prefix = paths.slice(0, idx).join('/')
458 const formattedItems = formatFolderItems(folderItems, prefix)
459 return {
460 id: null,
461 status: STORAGE_ROW_STATUS.READY,
462 name: idx === 0 ? state.selectedBucket.name : pathsWithEmptyPrefix[idx],
463 path: prefix,
464 items: formattedItems,
465 hasMoreItems: formattedItems.length === LIMIT,
466 isLoadingMoreItems: false,
467 }
468 })
469
470 // Package into columns and update this.columns
471 state.columns = formattedFolders
472
473 // Update openedFolders as well
474 const updatedOpenedFolders: StorageItem[] = paths.map((path, idx) => {
475 const folderInfo = find(formattedFolders[idx].items, { name: path })
476 // Folder doesnt exist, FE just scaffolds a "fake" folder
477 if (!folderInfo) {
478 return {
479 id: null,
480 name: path,
481 type: STORAGE_ROW_TYPES.FOLDER,
482 status: STORAGE_ROW_STATUS.READY,
483 metadata: null,
484 isCorrupted: false,
485 created_at: null,
486 updated_at: null,
487 last_accessed_at: null,
488 }
489 }
490 return folderInfo
491 })
492 state.openedFolders = updatedOpenedFolders
493 },
494
495 /**
496 * Check parent folder if its empty, if yes, reinstate .emptyFolderPlaceholder. Used when deleting folder or deleting files
497 */
498 validateParentFolderEmpty: async (parentFolderPrefix: string) => {
499 try {
500 const data = await listBucketObjects({
501 projectRef: state.projectRef,
502 bucketId: state.selectedBucket.id,
503 path: parentFolderPrefix,
504 options: {
505 limit: LIMIT,
506 offset: OFFSET,
507 sortBy: getSortOptions(),
508 },
509 })
510
511 if (data.length === 0) {
512 const prefixToPlaceholder = `${parentFolderPrefix}/${EMPTY_FOLDER_PLACEHOLDER_FILE_NAME}`
513 const client = await createProjectBrivenClient(state.projectRef, clientEndpoint)
514 await client.storage
515 .from(state.selectedBucket.name)
516 .upload(prefixToPlaceholder, new File([], EMPTY_FOLDER_PLACEHOLDER_FILE_NAME))
517 }
518 } catch (error) {}
519 },
520
521 deleteFolder: async (folder: StorageItemWithColumn) => {
522 try {
523 const isDeleteFolder = true
524 const files = await state.getAllItemsAlongFolder(folder)
525
526 if (files.length === 0) {
527 // [Joshen] This is to self-remediate orphan prefixes
528 await deleteBucketPrefix({
529 projectRef: state.projectRef,
530 connectionString: state.connectionString,
531 bucketId: state.selectedBucket.id,
532 prefix: folder.path,
533 })
534 } else {
535 await state.deleteFiles({ files: files as any[], isDeleteFolder })
536 }
537
538 state.popColumnAtIndex(folder.columnIndex)
539 state.popOpenedFoldersAtIndex(folder.columnIndex - 1)
540
541 if (folder.columnIndex > 0) {
542 const parentFolderPrefix = state.openedFolders
543 .slice(0, folder.columnIndex)
544 .map((folder) => folder.name)
545 .join('/')
546 if (parentFolderPrefix.length > 0)
547 await state.validateParentFolderEmpty(parentFolderPrefix)
548 }
549
550 await state.refetchAllOpenedFolders()
551 state.setSelectedItemsToDelete([])
552 toast.success(`Successfully deleted ${folder.name}`)
553 } catch (error: any) {
554 toast.error(`Failed to delete folder: ${error.message}`)
555 }
556 },
557
558 renameFolder: async ({
559 folder,
560 newName,
561 columnIndex,
562 onError,
563 }: {
564 folder: StorageItemWithColumn
565 newName: string
566 columnIndex: number
567 onError?: () => void
568 }) => {
569 const originalName = folder.name
570 if (originalName === newName) {
571 return state.updateRowStatus({
572 name: originalName,
573 status: STORAGE_ROW_STATUS.READY,
574 columnIndex,
575 })
576 }
577
578 const folderNameError = validateFolderName(newName)
579 if (folderNameError) {
580 onError?.()
581 return toast.error(folderNameError)
582 }
583
584 const toastId = toast(
585 <SonnerProgress progress={0} message={`Renaming folder to ${newName}`} />,
586 { closeButton: false, position: 'top-right' }
587 )
588
589 try {
590 state.updateRowStatus({
591 name: originalName,
592 status: STORAGE_ROW_STATUS.LOADING,
593 columnIndex,
594 updatedName: newName,
595 })
596 const files = await state.getAllItemsAlongFolder(folder)
597
598 let progress = 0
599 let failedFiles = 0
600 let retrySeconds = DEFAULT_RETRY_SECONDS
601
602 for (const file of files) {
603 const fromPath = `${file.prefix}/${file.name}`
604 const pathSegments = fromPath.split('/')
605 const toPath = pathSegments
606 .slice(0, columnIndex)
607 .concat(newName)
608 .concat(pathSegments.slice(columnIndex + 1))
609 .join('/')
610
611 let success = false
612 let isRateLimited = false
613
614 for (let attempt = 0; attempt < 3 && !success; attempt++) {
615 try {
616 if (attempt > 0) {
617 await new Promise<void>((resolve) => {
618 let seconds = retrySeconds
619 const interval = setInterval(() => {
620 toast(
621 <SonnerProgress
622 progress={Math.min(progress * 100, 100)}
623 message={`Renaming folder to ${newName}`}
624 description={`${isRateLimited ? 'API rate limited' : 'Error moving file'} - retrying in ${seconds} seconds (${attempt}/3)`}
625 />,
626 { id: toastId, closeButton: false, position: 'top-right', duration: Infinity }
627 )
628
629 seconds--
630 if (seconds <= 0) {
631 clearInterval(interval)
632 resolve()
633 }
634 }, 1000)
635 })
636 }
637
638 await moveStorageObject({
639 projectRef: state.projectRef,
640 bucketId: state.selectedBucket.id,
641 from: fromPath,
642 to: toPath,
643 })
644 success = true
645 } catch (error) {
646 if ((error as ResponseError).code === 429) {
647 isRateLimited = true
648 retrySeconds = RATE_LIMIT_RETRY_SECONDS
649 } else {
650 isRateLimited = false
651 retrySeconds = DEFAULT_RETRY_SECONDS
652 }
653
654 if (attempt === 2) failedFiles += 1
655 }
656 }
657
658 progress += 1 / files.length
659 toast(
660 <SonnerProgress
661 progress={Math.min(progress * 100, 100)}
662 message={`Renaming folder to ${newName}`}
663 />,
664 { id: toastId, closeButton: false, position: 'top-right', duration: Infinity }
665 )
666 }
667
668 if (failedFiles === 0) {
669 toast.success(`Successfully renamed folder to ${newName}`, {
670 id: toastId,
671 closeButton: true,
672 duration: SONNER_DEFAULT_DURATION,
673 })
674 } else {
675 toast.error(
676 <div>
677 <p>
678 Renamed folder to {newName} with {failedFiles} error{failedFiles > 1 ? 's' : ''}
679 </p>
680 <p className="text-foreground-light">
681 You may try again to rename the folder {originalName} to {newName}
682 </p>
683 </div>,
684 {
685 id: toastId,
686 closeButton: true,
687 duration: Infinity,
688 }
689 )
690 }
691
692 if (state.openedFolders[columnIndex]?.name === folder.name) {
693 state.setSelectedFilePreview(undefined)
694 state.popOpenedFoldersAtIndex(columnIndex - 1)
695 }
696 await state.refetchAllOpenedFolders()
697
698 // TODO: Should we invalidate the file preview cache when renaming folders?
699 } catch (e: any) {
700 toast.error(`Failed to rename folder to ${newName}: ${e.message}`, {
701 id: toastId,
702 closeButton: true,
703 duration: SONNER_DEFAULT_DURATION,
704 })
705 }
706 },
707
708 updateFolderAfterEdit: ({
709 folderName,
710 columnIndex,
711 status = STORAGE_ROW_STATUS.READY,
712 }: {
713 folderName: string
714 columnIndex?: number
715 status?: STORAGE_ROW_STATUS
716 }) => {
717 const columnIndex_ = columnIndex !== undefined ? columnIndex : state.getLatestColumnIndex()
718 const updatedColumns = state.columns.map((column, idx) => {
719 if (idx === columnIndex_) {
720 const updatedItems = column.items.map((item) => {
721 if (item.status === STORAGE_ROW_STATUS.EDITING) {
722 const currentTime = new Date().toISOString()
723 return {
724 ...item,
725 status,
726 name: folderName,
727 createdAt: currentTime,
728 lastAccessedAt: currentTime,
729 updatedAt: currentTime,
730 metadata: null,
731 id: null,
732 }
733 }
734 return item
735 })
736 return { ...column, items: updatedItems }
737 }
738 return column
739 })
740 state.columns = updatedColumns
741 },
742
743 openFolder: async (columnIndex: number, folder: StorageItem) => {
744 state.setSelectedFilePreview(undefined)
745 state.clearSelectedItems(columnIndex + 1)
746 state.popOpenedFoldersAtIndex(columnIndex - 1)
747 state.pushOpenedFolderAtIndex(folder, columnIndex)
748 await state.fetchFolderContents({
749 bucketId: state.selectedBucket.id,
750 folderId: folder.id,
751 folderName: folder.name,
752 index: columnIndex,
753 })
754 },
755
756 downloadFolder: async (folder: StorageItemWithColumn) => {
757 let progress = 0
758 const toastId = toast.loading('Retrieving files from folder...')
759
760 try {
761 const files = await state.getAllItemsAlongFolder(folder)
762
763 toast(
764 <SonnerProgress
765 progress={0}
766 message={`Downloading ${files.length} file${files.length > 1 ? 's' : ''}...`}
767 />,
768 { id: toastId, closeButton: false, position: 'top-right' }
769 )
770
771 // Pre-fetch all URLs in a single batch to avoid N management API calls
772 const filePaths = files.map((file) => `${file.prefix}/${file.name}`)
773 const urlByPath = new Map<string, string>()
774
775 if (state.selectedBucket.public) {
776 for (const filePath of filePaths) {
777 urlByPath.set(
778 filePath,
779 `${clientEndpoint}/storage/v1/object/public/${state.selectedBucket.id}/${filePath}`
780 )
781 }
782 } else {
783 const signedUrls = await signBucketObjects({
784 projectRef: state.projectRef,
785 bucketId: state.selectedBucket.id,
786 paths: filePaths,
787 expiresIn: 60 * 60, // 1 hour — enough for large folder downloads
788 })
789 for (const item of signedUrls) {
790 if (item.path && item.signedUrl) {
791 urlByPath.set(item.path, item.signedUrl)
792 }
793 }
794 }
795
796 const promises = files.map((file) => {
797 const fileMimeType = (file.metadata?.mimetype as string) ?? null
798 const filePath = `${file.prefix}/${file.name}`
799 return () => {
800 return new Promise<
801 | {
802 name: string
803 prefix: string
804 blob: Blob
805 }
806 | boolean
807 >(async (resolve) => {
808 try {
809 const url = urlByPath.get(filePath)
810 if (!url) throw new Error(`Failed to retrieve file ${filePath}`)
811
812 const response = await fetch(url)
813 if (!response.ok) throw new Error(`Failed to retrieve file ${filePath}`)
814 const data = await response.blob()
815
816 progress = progress + 1 / files.length
817
818 resolve({
819 name: file.name,
820 prefix: file.prefix,
821 blob: new Blob([data], { type: fileMimeType ?? data.type }),
822 })
823 } catch (error) {
824 console.error('Failed to download file', filePath)
825 resolve(false)
826 }
827 })
828 }
829 })
830
831 const batchedPromises = chunk(promises, 10)
832 const downloadedFiles = await batchedPromises.reduce(
833 async (previousPromise, nextBatch) => {
834 const previousResults = await previousPromise
835 const batchResults = await Promise.allSettled(nextBatch.map((batch) => batch()))
836 toast(
837 <SonnerProgress
838 progress={progress * 100}
839 message={`Downloading ${files.length} file${files.length > 1 ? 's' : ''}...`}
840 />,
841 { id: toastId, closeButton: false, position: 'top-right' }
842 )
843 return previousResults.concat(batchResults.map((x: any) => x.value).filter(Boolean))
844 },
845 Promise.resolve<
846 {
847 name: string
848 prefix: string
849 blob: Blob
850 }[]
851 >([])
852 )
853
854 const zipFileWriter = new BlobWriter('application/zip')
855 const zipWriter = new ZipWriter(zipFileWriter, { bufferedWrite: true })
856
857 if (downloadedFiles.length === 0) {
858 toast.error(`Failed to download files from "${folder.name}"`, {
859 id: toastId,
860 closeButton: true,
861 duration: SONNER_DEFAULT_DURATION,
862 })
863 }
864
865 downloadedFiles.forEach((file) => {
866 if (file.blob) zipWriter.add(`${file.prefix}/${file.name}`, new BlobReader(file.blob))
867 })
868
869 const blobURL = URL.createObjectURL(await zipWriter.close())
870 const link = document.createElement('a')
871 link.href = blobURL
872 link.setAttribute('download', `${folder.name}.zip`)
873 document.body.appendChild(link)
874 link.click()
875 link.parentNode?.removeChild(link)
876
877 toast.success(
878 downloadedFiles.length === files.length
879 ? `Successfully downloaded folder "${folder.name}"`
880 : `Downloaded folder "${folder.name}". However, ${
881 files.length - downloadedFiles.length
882 } files did not download successfully.`,
883 { id: toastId, closeButton: true, duration: SONNER_DEFAULT_DURATION }
884 )
885 } catch (error: any) {
886 toast.error(`Failed to download folder: ${error.message}`, {
887 id: toastId,
888 closeButton: true,
889 duration: SONNER_DEFAULT_DURATION,
890 })
891 }
892 },
893
894 /**
895 * Recursively returns a list of items along every directory within the specified base folder
896 * Each item has an extra property 'prefix' which has the prefix that leads to the item
897 * Used specifically for any operation that deals with every file along the folder
898 * e.g Delete folder, rename folder
899 */
900 getAllItemsAlongFolder: async (folder: {
901 name: string
902 columnIndex: number
903 prefix?: string
904 }): Promise<(StorageObject & { prefix: string })[]> => {
905 const items: (StorageObject & { prefix: string })[] = []
906
907 let hasError = false
908 let formattedPathToFolder = ''
909 const { name, columnIndex, prefix } = folder
910
911 if (prefix === undefined) {
912 const pathToFolder = state.openedFolders
913 .slice(0, columnIndex)
914 .map((folder) => folder.name)
915 .join('/')
916 formattedPathToFolder = pathToFolder.length > 0 ? `${pathToFolder}/${name}` : name
917 } else {
918 formattedPathToFolder = `${prefix}/${name}`
919 }
920
921 // [Joshen] limit is set to 10k to optimize reduction of requests, we've done some experiments
922 // that prove that the time to fetch all files in a folder reduces as the batch size increases
923 // 10k however, is the hard limit at the API level.
924 const options = {
925 limit: 10000,
926 offset: OFFSET,
927 sortBy: getSortOptions(),
928 }
929 let folderContents: StorageObject[] = []
930
931 for (;;) {
932 try {
933 const data = await listBucketObjects({
934 projectRef: state.projectRef,
935 bucketId: state.selectedBucket.id,
936 path: formattedPathToFolder,
937 options,
938 })
939 folderContents = folderContents.concat(data)
940 options.offset += options.limit
941 if ((data || []).length < options.limit) {
942 break
943 }
944 } catch (e) {
945 hasError = true
946 break
947 }
948 }
949
950 if (hasError) {
951 throw new Error('Failed to retrieve all files within folder')
952 }
953
954 const subfolders = folderContents?.filter((item) => item.id === null) ?? []
955 const folderItems = folderContents?.filter((item) => item.id !== null) ?? []
956
957 folderItems.forEach((item) => items.push({ ...item, prefix: formattedPathToFolder }))
958
959 const subFolderContents = await Promise.all(
960 subfolders.map((folder) =>
961 state.getAllItemsAlongFolder({ ...folder, columnIndex: 0, prefix: formattedPathToFolder })
962 )
963 )
964 subFolderContents.map((subfolderContent) => {
965 subfolderContent.map((item) => items.push(item))
966 })
967
968 return items
969 },
970
971 // ======== Files CRUD ========
972
973 uploadFiles: async ({
974 files,
975 columnIndex,
976 isDrop = false,
977 }: {
978 files: FileList | DataTransferItemList
979 columnIndex: number
980 isDrop?: boolean
981 }) => {
982 const queryClient = getQueryClient()
983 const storageConfiguration = queryClient
984 .getQueryCache()
985 .find({ queryKey: configKeys.storage(state.projectRef) })?.state.data as
986 | ProjectStorageConfigResponse
987 | undefined
988 const fileSizeLimit = storageConfiguration?.fileSizeLimit
989
990 const t1 = new Date()
991
992 const autofix = true
993 // We filter out any folders which are just '#' until we can properly encode such characters in the URL
994 const filesToUpload: (File & { path?: string })[] = isDrop
995 ? (await getFilesDataTransferItems(files as DataTransferItemList)).filter(
996 (file) => !file.path.includes('#/')
997 )
998 : Array.from(files as FileList)
999 const derivedColumnIndex = columnIndex === -1 ? state.getLatestColumnIndex() : columnIndex
1000
1001 const filesWithinUploadLimit =
1002 fileSizeLimit !== undefined
1003 ? filesToUpload.filter((file) => file.size <= fileSizeLimit)
1004 : filesToUpload
1005
1006 if (filesWithinUploadLimit.length < filesToUpload.length) {
1007 const numberOfFilesRejected = filesToUpload.length - filesWithinUploadLimit.length
1008 const { value, unit } = convertFromBytes(fileSizeLimit as number)
1009
1010 toast.error(
1011 <div className="flex flex-col gap-y-1">
1012 <p className="text-foreground">
1013 Failed to upload {numberOfFilesRejected} file{numberOfFilesRejected > 1 ? 's' : ''} as{' '}
1014 {numberOfFilesRejected > 1 ? 'their' : 'its'} size
1015 {numberOfFilesRejected > 1 ? 's are' : ' is'} beyond the global upload limit of{' '}
1016 {value} {unit}.
1017 </p>
1018 <p className="text-foreground-light">
1019 You can change the global file size upload limit in{' '}
1020 <InlineLink href={`/project/${state.projectRef}/storage/settings`}>
1021 Storage Settings
1022 </InlineLink>
1023 .
1024 </p>
1025 </div>,
1026 { duration: 8000 }
1027 )
1028
1029 if (numberOfFilesRejected === filesToUpload.length) return
1030 }
1031
1032 const filesWithNonZeroSize = filesWithinUploadLimit.filter((file) => file.size > 0)
1033 if (filesWithNonZeroSize.length < filesWithinUploadLimit.length) {
1034 const numberOfFilesRejected = filesWithinUploadLimit.length - filesWithNonZeroSize.length
1035 toast.error(
1036 <div className="flex flex-col gap-y-1">
1037 <p className="text-foreground">
1038 Failed to upload {numberOfFilesRejected} file{numberOfFilesRejected > 1 ? 's' : ''} as{' '}
1039 {numberOfFilesRejected > 1 ? 'their' : 'its'} size
1040 {numberOfFilesRejected > 1 ? 's are' : ' is'} 0.
1041 </p>
1042 </div>
1043 )
1044
1045 if (numberOfFilesRejected === filesWithinUploadLimit.length) return
1046 }
1047
1048 // If we're uploading a folder which name already exists in the same folder that we're uploading to
1049 // We sanitize the folder name and let all file uploads through. (This is only via drag drop)
1050 const topLevelFolders: string[] = (state.columns?.[derivedColumnIndex]?.items ?? [])
1051 .filter((item) => !item.id)
1052 .map((item) => item.name)
1053 const formattedFilesToUpload = filesWithinUploadLimit
1054 .filter((file) => file.name !== '.DS_Store')
1055 .map((file) => {
1056 // If the files are from clicking "Upload button", just take them as they are since users cannot
1057 // upload folders from clicking that button, only via drag drop
1058 if (!file.path) return file
1059
1060 const path = file.path.split('/')
1061 const topLevelFolder = path.length > 1 ? path[0] : null
1062 if (topLevelFolders.includes(topLevelFolder as string)) {
1063 const newTopLevelFolder = sanitizeNameForDuplicateInColumn(state, {
1064 name: topLevelFolder as string,
1065 autofix,
1066 columnIndex,
1067 })
1068 path[0] = newTopLevelFolder as string
1069 file.path = path.join('/')
1070 }
1071 return file
1072 })
1073
1074 state.uploadProgresses = new Array(formattedFilesToUpload.length).fill({
1075 percentage: 0,
1076 elapsed: 0,
1077 uploadSpeed: 0,
1078 remainingBytes: 0,
1079 remainingTime: 0,
1080 })
1081 const uploadedTopLevelFolders: string[] = []
1082 const numberOfFilesToUpload = formattedFilesToUpload.length
1083 let numberOfFilesUploadedSuccess = 0
1084 let numberOfFilesUploadedFail = 0
1085
1086 const pathToFile = state.openedFolders
1087 .slice(0, derivedColumnIndex)
1088 .map((folder) => folder.name)
1089 .join('/')
1090
1091 const toastId = state.onUploadProgress()
1092
1093 // Upload files in batches
1094 const promises = formattedFilesToUpload.map((file, index) => {
1095 const extension = file.name.split('.').pop()
1096 const metadata = {
1097 mimetype: (file.type || lookupMime(extension)) ?? '',
1098 size: file.size,
1099 } as StorageItemMetadata
1100 const fileOptions = { cacheControl: '3600', contentType: metadata.mimetype }
1101
1102 const isWithinFolder = (file?.path ?? '').split('/').length > 1
1103 const fileName = !isWithinFolder
1104 ? sanitizeNameForDuplicateInColumn(state, { name: file.name, autofix })
1105 : file.name
1106 const unsanitizedFormattedFileName =
1107 has(file, ['path']) && isWithinFolder ? file.path : fileName
1108 /**
1109 * Storage maintains a list of allowed characters, which excludes
1110 * characters such as the narrow no-break space used in Mac screenshots.
1111 * [Joshen] Am limiting to just replacing nbsp with a blank space instead of
1112 * all non-word characters per before
1113 * */
1114 const formattedFileName = (unsanitizedFormattedFileName ?? 'unknown').replaceAll(
1115 /\u{202F}/gu,
1116 ' '
1117 )
1118 const formattedPathToFile =
1119 pathToFile.length > 0
1120 ? `${pathToFile}/${formattedFileName}`
1121 : (formattedFileName as string)
1122
1123 if (isWithinFolder) {
1124 const topLevelFolder = file.path?.split('/')[0] || ''
1125 if (!uploadedTopLevelFolders.includes(topLevelFolder)) {
1126 state.addTempRow({
1127 type: STORAGE_ROW_TYPES.FOLDER,
1128 name: topLevelFolder,
1129 status: STORAGE_ROW_STATUS.LOADING,
1130 columnIndex: derivedColumnIndex,
1131 metadata,
1132 })
1133 uploadedTopLevelFolders.push(topLevelFolder)
1134 }
1135 } else {
1136 state.addTempRow({
1137 type: STORAGE_ROW_TYPES.FILE,
1138 name: fileName!,
1139 status: STORAGE_ROW_STATUS.LOADING,
1140 columnIndex: derivedColumnIndex,
1141 metadata,
1142 })
1143 }
1144
1145 let startingBytes = 0
1146 const bucketName = state.selectedBucket.name
1147
1148 return () => {
1149 return new Promise<void>(async (resolve, reject) => {
1150 const fileSizeInMB = file.size / (1024 * 1024)
1151 const startTime = Date.now()
1152
1153 let chunkSize: number
1154
1155 if (fileSizeInMB < 30) {
1156 chunkSize = 6 * 1024 * 1024 // 6MB
1157 } else if (fileSizeInMB < 100) {
1158 chunkSize = Math.floor(file.size / 8) // maximum 8 chunks, 12,5MB each
1159 } else if (fileSizeInMB < 500) {
1160 chunkSize = Math.floor(file.size / 10) // maximum 10 chunks, 50MB each
1161 } else if (fileSizeInMB < 1024) {
1162 chunkSize = Math.floor(file.size / 20) // maximum 20 chunks, 50MB each
1163 } else if (fileSizeInMB < 10 * 1024) {
1164 chunkSize = Math.floor(file.size / 30) // maximum 30 chunks, 333MB each
1165 } else {
1166 chunkSize = Math.floor(file.size / 50) // maximum 50 chunks
1167 }
1168
1169 // Max chunk size is 500MB
1170 chunkSize = Math.min(chunkSize, 500 * 1024 * 1024)
1171 const uploadDataDuringCreation = file.size <= chunkSize
1172
1173 const upload = new tus.Upload(file, {
1174 endpoint: state.resumableUploadUrl,
1175 retryDelays: [0, 200, 500, 1500, 3000, 5000],
1176 headers: {
1177 'x-source': 'briven-dashboard',
1178 },
1179 uploadDataDuringCreation: uploadDataDuringCreation,
1180 removeFingerprintOnSuccess: true,
1181 metadata: {
1182 bucketName,
1183 objectName: formattedPathToFile,
1184 ...fileOptions,
1185 },
1186 chunkSize,
1187 onBeforeRequest: async (req) => {
1188 try {
1189 // Use the shared temporary key for batch uploads
1190 // This checks if the key is still valid and refreshes if needed
1191 const { apiKey } = await getOrRefreshTemporaryApiKey(state.projectRef)
1192 req.setHeader('apikey', apiKey)
1193 if (!IS_PLATFORM) {
1194 req.setHeader('Authorization', `Bearer ${apiKey}`)
1195 }
1196 } catch (error) {
1197 throw error
1198 }
1199 },
1200 onShouldRetry(error) {
1201 const status = error.originalResponse ? error.originalResponse.getStatus() : 0
1202 const doNotRetryStatuses = [400, 403, 404, 409, 413, 415, 429]
1203
1204 return !doNotRetryStatuses.includes(status)
1205 },
1206 onError: (error) => {
1207 numberOfFilesUploadedFail += 1
1208 if (error instanceof tus.DetailedError) {
1209 const status = error.originalResponse?.getStatus()
1210
1211 switch (status) {
1212 case 415: {
1213 // Unsupported mime type
1214 toast.error(
1215 capitalize(
1216 error?.originalResponse?.getBody() ||
1217 `Failed to upload ${file.name}: ${metadata.mimetype} is not allowed`
1218 ),
1219 {
1220 description: `Allowed MIME types: ${state.selectedBucket.allowed_mime_types?.join(', ')}`,
1221 }
1222 )
1223 break
1224 }
1225 case 413: {
1226 // Payload too large
1227 toast.error(
1228 `Failed to upload ${file.name}: File size exceeds the bucket file size limit.`
1229 )
1230 break
1231 }
1232 case 409: {
1233 // Resource already exists
1234 toast.error(`Failed to upload ${file.name}: File name already exists.`)
1235 break
1236 }
1237 case 400: {
1238 const responseBody = error.originalResponse?.getBody()
1239 if (typeof responseBody === 'string') {
1240 if (responseBody.includes('Invalid key:')) {
1241 toast.error(`Failed to upload ${file.name}: File name is invalid.`)
1242 break
1243 }
1244
1245 if (responseBody.includes('Invalid Compact JWS')) {
1246 toast.error(`Failed to upload ${file.name}: Invalid Compact JWS.`)
1247 break
1248 }
1249 }
1250 // if it's not handled by the two ifs, fallthrough to the default case which shows the generic error message
1251 }
1252 default: {
1253 toast.error(`Failed to upload ${file.name}: ${error.message}`)
1254 break
1255 }
1256 }
1257 } else {
1258 toast.error(`Failed to upload ${file.name}: ${error.message}`)
1259 }
1260 reject(error)
1261 },
1262 onProgress: (bytesSent, bytesTotal) => {
1263 if (startingBytes === 0 && bytesSent > chunkSize) {
1264 startingBytes = bytesSent
1265 }
1266
1267 const percentage = bytesTotal === 0 ? 0 : bytesSent / bytesTotal
1268 const realBytesSent = bytesSent - startingBytes
1269 const elapsed = (Date.now() - startTime) / 1000 // in seconds
1270 const uploadSpeed = realBytesSent / elapsed // in bytes per second
1271 const remainingBytes = bytesTotal - realBytesSent
1272 const remainingTime = remainingBytes / uploadSpeed // in seconds
1273
1274 state.uploadProgresses[index] = {
1275 percentage,
1276 elapsed,
1277 uploadSpeed,
1278 remainingBytes,
1279 remainingTime,
1280 }
1281 state.onUploadProgress(toastId)
1282 },
1283 onSuccess() {
1284 numberOfFilesUploadedSuccess += 1
1285 resolve()
1286 },
1287 })
1288
1289 if (!Array.isArray(state.abortUploadCallbacks[toastId])) {
1290 state.abortUploadCallbacks[toastId] = []
1291 }
1292 state.abortUploadCallbacks[toastId].push(() => {
1293 try {
1294 upload.abort(true)
1295 } catch (error) {
1296 // Ignore error
1297 }
1298 reject(new Error('Upload aborted by user'))
1299 })
1300
1301 // Check if there are any previous uploads to continue.
1302 return upload.findPreviousUploads().then((previousUploads) => {
1303 // Found previous uploads so we select the first one.
1304 if (previousUploads.length) {
1305 upload.resumeFromPreviousUpload(previousUploads[0])
1306 }
1307
1308 upload.start()
1309 })
1310 })
1311 }
1312 })
1313
1314 // For file uploads specifically, we have to lower the batch size for now as the client side
1315 // is just unable to handle such volumes of transfer
1316 // [Joshen] I realised this can be simplified with just a vanilla for loop, no need for reduce
1317 // Just take note, but if it's working fine, then it's okay
1318 const batchedPromises = chunk(promises, 10)
1319 try {
1320 await batchedPromises.reduce(async (previousPromise, nextBatch) => {
1321 await previousPromise
1322 await Promise.allSettled(nextBatch.map((batch) => batch()))
1323 state.onUploadProgress(toastId)
1324 }, Promise.resolve())
1325
1326 if (numberOfFilesUploadedSuccess > 0) {
1327 await deleteBucketObject({
1328 projectRef: state.projectRef,
1329 bucketId: state.selectedBucket.id,
1330 paths: [`${pathToFile}/${EMPTY_FOLDER_PLACEHOLDER_FILE_NAME}`],
1331 })
1332 }
1333
1334 await state.refetchAllOpenedFolders()
1335
1336 if (
1337 numberOfFilesToUpload === 0 ||
1338 (numberOfFilesUploadedSuccess === 0 && numberOfFilesUploadedFail === 0)
1339 ) {
1340 toast.dismiss(toastId)
1341 } else if (numberOfFilesUploadedFail === numberOfFilesToUpload) {
1342 if (numberOfFilesToUpload === 1) {
1343 // [Joshen] We'd already be showing a toast when the upload files, so this is to prevent a
1344 // duplicate error toast if its only one file that's getting uploaded
1345 toast.dismiss(toastId)
1346 } else {
1347 toast.error(
1348 `Failed to upload ${numberOfFilesToUpload} file${numberOfFilesToUpload > 1 ? 's' : ''}!`,
1349 { id: toastId, closeButton: true, duration: SONNER_DEFAULT_DURATION }
1350 )
1351 }
1352 } else if (numberOfFilesUploadedSuccess === numberOfFilesToUpload) {
1353 toast.success(
1354 `Successfully uploaded ${numberOfFilesToUpload} file${
1355 numberOfFilesToUpload > 1 ? 's' : ''
1356 }!`,
1357 { id: toastId, closeButton: true, duration: SONNER_DEFAULT_DURATION }
1358 )
1359 } else {
1360 toast.success(
1361 `Successfully uploaded ${numberOfFilesUploadedSuccess} out of ${numberOfFilesToUpload} file${
1362 numberOfFilesToUpload > 1 ? 's' : ''
1363 }!`,
1364 { id: toastId, closeButton: true, duration: SONNER_DEFAULT_DURATION }
1365 )
1366 }
1367 } catch (e) {
1368 toast.error('Failed to upload files', {
1369 id: toastId,
1370 closeButton: true,
1371 duration: SONNER_DEFAULT_DURATION,
1372 })
1373 }
1374
1375 const t2 = new Date()
1376 console.log(
1377 `Total time taken for ${formattedFilesToUpload.length} files: ${((t2 as any) - (t1 as any)) / 1000} seconds`
1378 )
1379 },
1380
1381 moveFiles: async (newPathToFile: string) => {
1382 const newPaths = compact(newPathToFile.split('/'))
1383 const formattedNewPathToFile = newPaths.join('/')
1384 let numberOfFilesMovedFail = 0
1385 state.clearSelectedItems()
1386
1387 const toastId = toast(
1388 `Moving ${state.selectedItemsToMove.length} file${state.selectedItemsToMove.length > 1 ? 's' : ''}...`,
1389 {
1390 description: STORAGE_PROGRESS_INFO_TEXT,
1391 duration: Infinity,
1392 }
1393 )
1394
1395 await Promise.all(
1396 state.selectedItemsToMove.map(async (item) => {
1397 const pathToFile = state.openedFolders
1398 .slice(0, item.columnIndex)
1399 .map((folder) => folder.name)
1400 .join('/')
1401
1402 const fromPath = pathToFile.length > 0 ? `${pathToFile}/${item.name}` : item.name
1403 const toPath =
1404 newPathToFile.length > 0 ? `${formattedNewPathToFile}/${item.name}` : item.name
1405
1406 try {
1407 await moveStorageObject({
1408 projectRef: state.projectRef,
1409 bucketId: state.selectedBucket.id,
1410 from: fromPath,
1411 to: toPath,
1412 })
1413 } catch (error: any) {
1414 numberOfFilesMovedFail += 1
1415 toast.error(error.message)
1416 }
1417 })
1418 )
1419
1420 if (numberOfFilesMovedFail === state.selectedItemsToMove.length) {
1421 toast.error('Failed to move files')
1422 } else {
1423 toast(
1424 `Successfully moved ${
1425 state.selectedItemsToMove.length - numberOfFilesMovedFail
1426 } files to ${formattedNewPathToFile.length > 0 ? formattedNewPathToFile : 'the root of your bucket'}`
1427 )
1428 }
1429
1430 toast.dismiss(toastId)
1431
1432 // TODO: invalidate the file preview cache when moving files
1433 await state.refetchAllOpenedFolders()
1434 state.setSelectedItemsToMove([])
1435 },
1436
1437 deleteFiles: async ({
1438 files,
1439 isDeleteFolder = false,
1440 }: {
1441 files: (StorageItemWithColumn & { prefix?: string })[]
1442 isDeleteFolder?: boolean
1443 }) => {
1444 state.setSelectedFilePreview(undefined)
1445
1446 // If every file has the 'prefix' property, then just construct the prefix
1447 // directly (from delete folder). Otherwise go by the opened folders.
1448 const prefixes = !files.some((f) => f.prefix)
1449 ? files.map((file) => {
1450 const { name, columnIndex } = file
1451 const pathToFile = state.openedFolders
1452 .slice(0, columnIndex)
1453 .map((folder) => folder.name)
1454 .join('/')
1455 state.updateRowStatus({ name, status: STORAGE_ROW_STATUS.LOADING, columnIndex })
1456 return pathToFile.length > 0 ? `${pathToFile}/${name}` : name
1457 })
1458 : files.map((file) => `${file.prefix}/${file.name}`)
1459
1460 state.clearSelectedItems()
1461
1462 const toastId = toast.loading(`Deleting ${prefixes.length} file(s)...`)
1463
1464 try {
1465 await deleteBucketObject({
1466 projectRef: state.projectRef,
1467 bucketId: state.selectedBucket.id,
1468 paths: prefixes,
1469 })
1470
1471 if (!isDeleteFolder) {
1472 // If parent folders are empty, reinstate .emptyFolderPlaceholder to persist them
1473 const parentFolderPrefixes = uniq(
1474 prefixes.map((prefix) => {
1475 const segments = prefix.split('/')
1476 return segments.slice(0, segments.length - 1).join('/')
1477 })
1478 )
1479 await Promise.all(
1480 parentFolderPrefixes.map((prefix) => state.validateParentFolderEmpty(prefix))
1481 )
1482
1483 toast.success(`Successfully deleted ${prefixes.length} file(s)`, {
1484 id: toastId,
1485 closeButton: true,
1486 duration: SONNER_DEFAULT_DURATION,
1487 description: undefined,
1488 })
1489 await state.refetchAllOpenedFolders()
1490 state.setSelectedItemsToDelete([])
1491 } else {
1492 toast.dismiss(toastId)
1493 }
1494 } catch (err) {
1495 if (!isDeleteFolder) {
1496 toast.error(`Failed to delete ${prefixes.length} file(s)`, {
1497 id: toastId,
1498 closeButton: true,
1499 duration: SONNER_DEFAULT_DURATION,
1500 description: (err as ResponseError).message,
1501 })
1502
1503 if (!files.some((f) => f.prefix)) {
1504 files.forEach((file) => {
1505 const { name, columnIndex } = file
1506 state.updateRowStatus({ name, status: STORAGE_ROW_STATUS.READY, columnIndex })
1507 })
1508 }
1509 } else {
1510 toast.dismiss(toastId)
1511 throw err
1512 }
1513 }
1514 },
1515
1516 downloadFile: async (file: StorageItemWithColumn, showToast = true) => {
1517 if (!file.path) {
1518 toast.error('Failed to download: Unable to find path to file')
1519 return false
1520 }
1521
1522 const fileName: string = file.name
1523 const fileMimeType = file?.metadata?.mimetype ?? undefined
1524
1525 const toastId = showToast ? toast.loading(`Retrieving ${fileName}...`) : undefined
1526
1527 try {
1528 const url = await fetchFileUrl(
1529 file.path,
1530 state.projectRef,
1531 state.selectedBucket.id,
1532 state.selectedBucket.public
1533 )
1534 const response = await fetch(url)
1535 if (!response.ok) throw new Error(`Failed to retrieve file ${file.path}`)
1536 const data = await response.blob()
1537
1538 const newBlob = new Blob([data], { type: fileMimeType ?? data.type })
1539 const blobUrl = window.URL.createObjectURL(newBlob)
1540 const link = document.createElement('a')
1541 link.href = blobUrl
1542 link.setAttribute('download', `${fileName}`)
1543 document.body.appendChild(link)
1544 link.click()
1545 link.parentNode?.removeChild(link)
1546 window.URL.revokeObjectURL(blobUrl)
1547
1548 if (toastId) {
1549 toast.success(`Downloading ${fileName}`, {
1550 id: toastId,
1551 closeButton: true,
1552 duration: SONNER_DEFAULT_DURATION,
1553 })
1554 }
1555 return true
1556 } catch (err) {
1557 if (toastId) {
1558 toast.error(`Failed to download ${fileName}`, {
1559 id: toastId,
1560 closeButton: true,
1561 duration: SONNER_DEFAULT_DURATION,
1562 })
1563 }
1564 return false
1565 }
1566 },
1567
1568 downloadFileAsBlob: async (file: StorageItemWithColumn) => {
1569 if (!file.path) return false
1570
1571 try {
1572 const url = await fetchFileUrl(
1573 file.path,
1574 state.projectRef,
1575 state.selectedBucket.id,
1576 state.selectedBucket.public
1577 )
1578 const response = await fetch(url)
1579 if (!response.ok) throw new Error(`Failed to retrieve file ${file.path}`)
1580 const data = await response.blob()
1581
1582 const fileMimeType = file?.metadata?.mimetype ?? undefined
1583 return { name: file.name, blob: new Blob([data], { type: fileMimeType ?? data.type }) }
1584 } catch (err) {
1585 console.error('Failed to download file', file.path)
1586 return false
1587 }
1588 },
1589
1590 downloadSelectedFiles: async (files: StorageItemWithColumn[]) => {
1591 const lowestColumnIndex = Math.min(...files.map((file) => file.columnIndex))
1592
1593 const formattedFilesWithPrefix: any[] = files.map((file) => {
1594 const { name, columnIndex } = file
1595 const pathToFile = state.openedFolders
1596 .slice(lowestColumnIndex, columnIndex)
1597 .map((folder) => folder.name)
1598 .join('/')
1599 const formattedPathToFile = pathToFile.length > 0 ? `${pathToFile}/${name}` : name
1600 return { ...file, formattedPathToFile }
1601 })
1602
1603 let progress = 0
1604 const toastId = toast.loading(
1605 `Downloading ${files.length} file${files.length > 1 ? 's' : ''}...`
1606 )
1607
1608 const promises = formattedFilesWithPrefix.map((file) => {
1609 return () => {
1610 return new Promise<{ name: string; blob: Blob } | boolean>(async (resolve) => {
1611 const data = await state.downloadFileAsBlob(file)
1612 progress = progress + 1 / formattedFilesWithPrefix.length
1613 if (isObject(data)) {
1614 resolve({ ...data, name: file.formattedPathToFile })
1615 }
1616 resolve(false)
1617 })
1618 }
1619 })
1620
1621 // Increase batch size to 50 since Storage API doesn't throttle like Management API
1622 const batchedPromises = chunk(promises, 50)
1623 const downloadedFiles = await batchedPromises.reduce(async (previousPromise, nextBatch) => {
1624 const previousResults = await previousPromise
1625 const batchResults = await Promise.allSettled(nextBatch.map((batch) => batch()))
1626 toast(
1627 <SonnerProgress
1628 progress={progress * 100}
1629 message={`Downloading ${files.length} file${files.length > 1 ? 's' : ''}...`}
1630 />,
1631 { id: toastId, closeButton: false, position: 'top-right' }
1632 )
1633 return previousResults.concat(batchResults.map((x: any) => x.value).filter(Boolean))
1634 }, Promise.resolve<{ name: string; blob: Blob }[]>([]))
1635
1636 const zipFileWriter = new BlobWriter('application/zip')
1637 const zipWriter = new ZipWriter(zipFileWriter, { bufferedWrite: true })
1638 downloadedFiles.forEach((file) => {
1639 zipWriter.add(file.name, new BlobReader(file.blob))
1640 })
1641
1642 const blobURL = URL.createObjectURL(await zipWriter.close())
1643 const link = document.createElement('a')
1644 link.href = blobURL
1645 link.setAttribute('download', `briven-files.zip`)
1646 document.body.appendChild(link)
1647 link.click()
1648 link.parentNode?.removeChild(link)
1649
1650 toast.success(`Successfully downloaded ${downloadedFiles.length} files`, {
1651 id: toastId,
1652 closeButton: true,
1653 duration: SONNER_DEFAULT_DURATION,
1654 })
1655 },
1656
1657 renameFile: async (file: StorageItem, newName: string, columnIndex: number) => {
1658 const originalName = file.name
1659 if (originalName === newName || newName.length === 0) {
1660 state.updateRowStatus({ name: originalName, status: STORAGE_ROW_STATUS.READY, columnIndex })
1661 } else {
1662 state.updateRowStatus({
1663 name: originalName,
1664 status: STORAGE_ROW_STATUS.LOADING,
1665 columnIndex,
1666 updatedName: newName,
1667 })
1668 const pathToFile = getPathAlongFoldersToIndex(state, columnIndex)
1669
1670 const fromPath = pathToFile.length > 0 ? `${pathToFile}/${originalName}` : originalName
1671 const toPath = pathToFile.length > 0 ? `${pathToFile}/${newName}` : newName
1672
1673 try {
1674 await moveStorageObject({
1675 projectRef: state.projectRef,
1676 bucketId: state.selectedBucket.id,
1677 from: fromPath,
1678 to: toPath,
1679 })
1680
1681 toast.success(`Successfully renamed "${originalName}" to "${newName}"`)
1682
1683 // TODO: Should we invalidate the file preview cache when renaming files?
1684
1685 if (state.selectedFilePreview?.name === originalName) {
1686 const { previewUrl, ...fileData } = file as any
1687 state.setSelectedFilePreview({ ...fileData, name: newName })
1688 }
1689
1690 await state.refetchAllOpenedFolders()
1691 } catch (error: any) {
1692 toast.error(`Failed to rename file: ${error.message}`)
1693 state.updateRowStatus({
1694 name: originalName,
1695 status: STORAGE_ROW_STATUS.READY,
1696 columnIndex,
1697 })
1698 }
1699 }
1700 },
1701
1702 // ======== UI Helper functions ========
1703
1704 selectRangeItems: (columnIndex: number, toItemIndex: number) => {
1705 const columnItems = state.columns[columnIndex].items
1706 const toItem = columnItems[toItemIndex]
1707 const selectedItemIds = state.selectedItems.map((item) => item.id)
1708 const lastSelectedItemId = selectedItemIds[selectedItemIds.length - 1]
1709 const lastSelectedItemIndex = findIndex(columnItems, { id: lastSelectedItemId })
1710
1711 // Get the start and end index of the range to select
1712 const start = Math.min(toItemIndex, lastSelectedItemIndex)
1713 const end = Math.max(toItemIndex, lastSelectedItemIndex)
1714
1715 // Get the range to select and reverse the order if necessary
1716 const rangeToSelect = columnItems
1717 .slice(start, end + 1)
1718 // we need `columnIndex` in all item of `selectedItems`
1719 .map((item) => ({ ...item, columnIndex }))
1720 if (toItemIndex < lastSelectedItemIndex) {
1721 rangeToSelect.reverse()
1722 }
1723
1724 if (selectedItemIds.includes(toItem.id)) {
1725 const rangeToDeselectIds = rangeToSelect.map((item) => item.id)
1726 // Deselect all items within the selection range
1727 state.setSelectedItems(
1728 state.selectedItems.filter(
1729 (item) => item.id === toItem.id || !rangeToDeselectIds.includes(item.id)
1730 )
1731 )
1732 } else {
1733 // Select items within the range
1734 state.setSelectedItems(uniqBy(state.selectedItems.concat(rangeToSelect), 'id'))
1735 }
1736 },
1737
1738 addTempRow: ({
1739 type,
1740 name,
1741 status,
1742 columnIndex,
1743 metadata,
1744 isPrepend = false,
1745 }: {
1746 type: STORAGE_ROW_TYPES
1747 name: string
1748 status: STORAGE_ROW_STATUS
1749 columnIndex: number
1750 metadata: StorageItemMetadata | null
1751 isPrepend?: boolean
1752 }) => {
1753 const updatedColumns = state.columns.map((column, idx) => {
1754 if (idx === columnIndex) {
1755 const tempRow = { type, name, status, metadata } as StorageItem
1756 const updatedItems = isPrepend
1757 ? [tempRow].concat(column.items)
1758 : column.items.concat([tempRow])
1759 return { ...column, items: updatedItems }
1760 }
1761 return column
1762 })
1763 state.columns = updatedColumns
1764 },
1765
1766 removeTempRows: (columnIndex: number) => {
1767 const updatedColumns = state.columns.map((column, idx) => {
1768 if (idx === columnIndex) {
1769 const updatedItems = column.items.filter((item) => has(item, 'id'))
1770 return { ...column, items: updatedItems }
1771 }
1772 return column
1773 })
1774 state.columns = updatedColumns
1775 },
1776
1777 onUploadProgress: (toastId?: string | number) => {
1778 const totalFiles = state.uploadProgresses.length
1779 const progress =
1780 (state.uploadProgresses.reduce((acc, { percentage }) => acc + percentage, 0) / totalFiles) *
1781 100
1782 const remainingTime = calculateTotalRemainingTime(state.uploadProgresses)
1783
1784 return toast(
1785 <SonnerProgress
1786 progress={progress}
1787 message={`Uploading ${totalFiles} file${totalFiles > 1 ? 's' : ''}...`}
1788 progressPrefix={`${remainingTime && !isNaN(remainingTime) && isFinite(remainingTime) && remainingTime !== 0 ? `${formatTime(remainingTime)} remaining – ` : ''}`}
1789 action={
1790 toastId && (
1791 <Button
1792 size="tiny"
1793 type="default"
1794 className="ml-6"
1795 onClick={() => state.abortUploads(toastId)}
1796 >
1797 Cancel
1798 </Button>
1799 )
1800 }
1801 />,
1802 { id: toastId, closeButton: false, position: 'top-right', duration: Infinity }
1803 )
1804 },
1805
1806 updateRowStatus: ({
1807 name,
1808 status,
1809 columnIndex,
1810 updatedName,
1811 }: {
1812 name: string
1813 status: STORAGE_ROW_STATUS
1814 columnIndex?: number
1815 updatedName?: string
1816 }) => {
1817 const columnIndex_ = columnIndex !== undefined ? columnIndex : state.getLatestColumnIndex()
1818 const updatedColumns = state.columns.map((column, idx) => {
1819 if (idx === columnIndex_) {
1820 const updatedColumnItems = column.items.map((item) => {
1821 return item.name === name
1822 ? {
1823 ...item,
1824 status,
1825 ...(updatedName && { name: updatedName }),
1826 }
1827 : item
1828 })
1829 return { ...column, items: updatedColumnItems }
1830 }
1831 return column
1832 })
1833 state.columns = updatedColumns
1834 },
1835 })
1836
1837 return state
1838}
1839
1840export type StorageExplorerState = ReturnType<typeof createStorageExplorerState>
1841
1842const DEFAULT_STATE_CONFIG = {
1843 projectRef: '',
1844 connectionString: '',
1845 resumableUploadUrl: '',
1846 clientEndpoint: '',
1847 bucket: {} as Bucket,
1848}
1849
1850const StorageExplorerStateContext = createContext<StorageExplorerState>(
1851 createStorageExplorerState(DEFAULT_STATE_CONFIG)
1852)
1853
1854export const StorageExplorerStateContextProvider = ({ children }: PropsWithChildren) => {
1855 const { data: project } = useSelectedProjectQuery()
1856 const { data: bucket } = useSelectedBucket()
1857 const isPaused = project?.status === PROJECT_STATUS.INACTIVE
1858
1859 const [state, setState] = useState(() => createStorageExplorerState(DEFAULT_STATE_CONFIG))
1860 const stateRef = useLatest(state)
1861
1862 const {
1863 storageEndpoint,
1864 hostEndpoint,
1865 isSuccess: isSuccessSettings,
1866 } = useProjectApiUrl({ projectRef: project?.ref })
1867
1868 // [Joshen] JFYI opting with the useEffect here as the storage explorer state was being loaded
1869 // before the project details were ready, hence the store kept returning project ref as undefined
1870 // Can be verified when we're saving the storage explorer preferences into local storage, that ref is undefined
1871 // So the useEffect here is to make sure that the project ref is loaded into the state properly
1872 // Although I'd be keen to re-investigate this to see if we can remove this
1873 useEffect(() => {
1874 const hasDataReady = !!project && !!bucket
1875 const storeAlreadyLoaded = state.projectRef === project?.ref
1876
1877 if (!isPaused && hasDataReady && !storeAlreadyLoaded && isSuccessSettings) {
1878 const clientEndpoint = storageEndpoint ?? hostEndpoint ?? ''
1879 const resumableUploadUrl = `${clientEndpoint}/storage/v1/upload/resumable`
1880
1881 setState(
1882 createStorageExplorerState({
1883 projectRef: project.ref,
1884 connectionString: project.connectionString ?? '',
1885 bucket,
1886 resumableUploadUrl,
1887 clientEndpoint,
1888 })
1889 )
1890 }
1891 }, [
1892 state.projectRef,
1893 project,
1894 stateRef,
1895 isPaused,
1896 hostEndpoint,
1897 storageEndpoint,
1898 isSuccessSettings,
1899 bucket,
1900 ])
1901
1902 return (
1903 <StorageExplorerStateContext.Provider value={state}>
1904 {children}
1905 </StorageExplorerStateContext.Provider>
1906 )
1907}
1908
1909export function useStorageExplorerStateSnapshot(options?: Parameters<typeof useSnapshot>[1]) {
1910 const state = useContext(StorageExplorerStateContext)
1911 return useSnapshot(state, options) as StorageExplorerState
1912}