Interface
import type { DeleteObjectsOptions } from "../types/cloud";
interface DataSource {
source: string;
uri?: string;
index?: number;
limit?: number;
query?: unknown;
postQuery?: string | (...args: unknown[]) => unknown;
preRender?: string | (...args: unknown[]) => unknown;
whenEmpty?: string | (...args: unknown[]) => unknown;
removeEmpty?: boolean;
ignoreCache?: boolean | number | number[];
transactionState?: number;
transactionFail?: boolean;
}
interface DbDataSource extends DataSource {
name?: string;
table?: string;
credential?: string | unknown;
options?: unknown;
update?: unknown;
flat?: number;
willAbort?: boolean;
streamRow?: boolean | string | ((row: unknown) => Error | false | void) | null;
}
interface CloudDatabase extends DbDataSource {
source: "cloud";
service: string;
id?: string;
params?: unknown;
}
import type { CopyObjectAction } from "../types/cloud";
interface CloudStorage {
service: string;
credential: unknown;
bucket?: string;
admin?: CloudStorageAdmin;
upload?: CloudStorageUpload;
download?: CloudStorageDownload;
}
interface CloudStorageAdmin extends CloudStorageACL {
emptyBucket?: DeleteObjectsOptions | boolean;
configBucket?: {
create?: unknown;
policy?: unknown;
website?: {
indexPage?: string;
errorPage?: string;
indexPath?: string;
errorPath?: string;
};
retentionPolicy?: unknown;
};
/** @deprecated */
recursive?: boolean;
preservePath?: boolean;
}
interface CloudStorageAction {
pathname?: string;
filename?: string;
minStreamSize?: number | string;
chunkSize?: number | string;
chunkLimit?: number;
flags?: number;
active?: boolean;
overwrite?: boolean;
admin?: CloudStorageAdmin;
}
interface CloudStorageUpload extends CloudStorageACL, CloudStorageAction, CopyObjectAction {
buffer?: Buffer | null;
contentType?: string;
metadata?: Record<string, string>;
tags?: Record<string, string> | false;
options?: unknown;
fileGroup?: [Buffer | string, string, string?][];
descendantsGroup?: string[];
localStorage?: boolean;
endpoint?: string;
all?: boolean;
}
interface CloudStorageACL {
publicRead?: boolean | 0;
acl?: string;
}
interface CloudStorageDownload extends CloudStorageAction, CopyObjectAction {
keyname?: string;
versionId?: string;
options?: unknown;
deleteObject?: unknown;
waitStatus?: boolean;
}
Changelog
Added in version 0.14.0:
CloudStorageUpload property descendantsGroup for file extensions which are not renamed if the parent file is renamed was implemented.
Added in version 0.13.0:
CloudStorageUpload property copyObject | copyObject[] for alternate bucket location was implemented.
CloudStorageDownload property copyObject | copyObject[] for emulating move/rename and delete semantics was implemented.
Added in version 0.11.0:
CloudStorageAdmin property emptyBucket for directory listing with DeleteObjectsOptions was amended.
Deprecated since version 0.11.0:
CloudStorageAdmin property recursive for directory traversal was replaced with emptyBucket as DeleteObjectsOptions.
Added in version 0.9.0:
CloudStorageAction property chunkSize | chunkLimit for parallel multipart operations were created.
CloudStorageDownload property options for the provider client was created.
CloudStorageDownload property keyname for file to be downloaded and subsequently renamed to filename.
Admin
Auth
Internal use of these libraries that do not require credentials validation during service client API initialization can disable this behavior through settings. There are also cases where an unsupported authorization scheme is necessary which has not been implemented.
Caution
These are global settings and affect every connection per service.
{
"settings": {
"aws": {
"auth": {
"storage": true, // Default behavior
"database": false // Explicit to disable
}
}
}
}
Storage
Service |
CLOUD_UPLOAD_STREAM |
CLOUD_UPLOAD_CHUNK |
CLOUD_DOWNLOAD_CHUNK |
|---|---|---|---|
aws |
X |
X |
|
aws-v3 |
X |
X |
|
azure |
X |
X |
X |
gcp |
X |
X |
X |
oci |
X |
X |
Caution
Setting process.env.EMC_CLOUD_UPLOAD_BUFFER = "true" will enable the legacy behavior for Document based uploads.
Stream
Streaming was enabled by default due to its lower memory usage requirements. It is slower for small file transfers which is typical for a static web page.
Tip
Setting upload.minStreamSize = -1 will disable streaming for the current request.
const aws = require("@pi-r/aws");
aws.CLOUD_UPLOAD_STREAM = false;
Warning
Reading a buffer from disk has 2gb file size limit.
Chunk
Parallel transfers were enabled by default to accommodate large files. The old behavior is used when chunkSize is empty and will open one request per file.
const azure = require("@pi-r/azure");
azure.CLOUD_UPLOAD_CHUNK = false;
azure.CLOUD_DOWNLOAD_CHUNK = false;
Note
Chunking is only active when the upload file size is greater than chunkSize.