datasets

Creates, updates, deletes, gets or lists a datasets resource.

Overview

Name	datasets
Type	Resource
Id	google.aiplatform.datasets

Fields

The following fields are returned by SELECT queries:

get

Name	Datatype	Description
name	string	Output only. Identifier. The resource name of the Dataset. Format: projects/{project}/locations/{location}/datasets/{dataset}
createTime	string (google-datetime)	Output only. Timestamp when this Dataset was created.
dataItemCount	string (int64)	Output only. The number of DataItems in this Dataset. Only apply for non-structured Dataset.
description	string	The description of the Dataset.
displayName	string	Required. The user-defined name of the Dataset. The name can be up to 128 characters long and can consist of any UTF-8 characters.
encryptionSpec	object	Customer-managed encryption key spec for a Dataset. If set, this Dataset and all sub-resources of this Dataset will be secured by this key. (id: GoogleCloudAiplatformV1EncryptionSpec)
etag	string	Used to perform consistent read-modify-write updates. If not set, a blind "overwrite" update happens.
labels	object	The labels with user-defined metadata to organize your Datasets. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. No more than 64 user labels can be associated with one Dataset (System labels are excluded). See https://goo.gl/xmQnxf for more information and examples of labels. System reserved label keys are prefixed with "aiplatform.googleapis.com/" and are immutable. Following system labels exist for each Dataset: * "aiplatform.googleapis.com/dataset_metadata_schema": output only, its value is the metadata_schema's title.
metadata	any	Required. Additional information about the Dataset.
metadataArtifact	string	Output only. The resource name of the Artifact that was created in MetadataStore when creating the Dataset. The Artifact resource name pattern is projects/{project}/locations/{location}/metadataStores/{metadata_store}/artifacts/{artifact}.
metadataSchemaUri	string	Required. Points to a YAML file stored on Google Cloud Storage describing additional information about the Dataset. The schema is defined as an OpenAPI 3.0.2 Schema Object. The schema files that can be used here are found in gs://google-cloud-aiplatform/schema/dataset/metadata/.
modelReference	string	Optional. Reference to the public base model last used by the dataset. Only set for prompt datasets.
satisfiesPzi	boolean	Output only. Reserved for future use.
satisfiesPzs	boolean	Output only. Reserved for future use.
savedQueries	array	All SavedQueries belong to the Dataset will be returned in List/Get Dataset response. The annotation_specs field will not be populated except for UI cases which will only use annotation_spec_count. In CreateDataset request, a SavedQuery is created together if this field is set, up to one SavedQuery can be set in CreateDatasetRequest. The SavedQuery should not contain any AnnotationSpec.
updateTime	string (google-datetime)	Output only. Timestamp when this Dataset was last updated.

list

Name	Datatype	Description
name	string	Output only. Identifier. The resource name of the Dataset. Format: projects/{project}/locations/{location}/datasets/{dataset}
createTime	string (google-datetime)	Output only. Timestamp when this Dataset was created.
dataItemCount	string (int64)	Output only. The number of DataItems in this Dataset. Only apply for non-structured Dataset.
description	string	The description of the Dataset.
displayName	string	Required. The user-defined name of the Dataset. The name can be up to 128 characters long and can consist of any UTF-8 characters.
encryptionSpec	object	Customer-managed encryption key spec for a Dataset. If set, this Dataset and all sub-resources of this Dataset will be secured by this key. (id: GoogleCloudAiplatformV1EncryptionSpec)
etag	string	Used to perform consistent read-modify-write updates. If not set, a blind "overwrite" update happens.
labels	object	The labels with user-defined metadata to organize your Datasets. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. No more than 64 user labels can be associated with one Dataset (System labels are excluded). See https://goo.gl/xmQnxf for more information and examples of labels. System reserved label keys are prefixed with "aiplatform.googleapis.com/" and are immutable. Following system labels exist for each Dataset: * "aiplatform.googleapis.com/dataset_metadata_schema": output only, its value is the metadata_schema's title.
metadata	any	Required. Additional information about the Dataset.
metadataArtifact	string	Output only. The resource name of the Artifact that was created in MetadataStore when creating the Dataset. The Artifact resource name pattern is projects/{project}/locations/{location}/metadataStores/{metadata_store}/artifacts/{artifact}.
metadataSchemaUri	string	Required. Points to a YAML file stored on Google Cloud Storage describing additional information about the Dataset. The schema is defined as an OpenAPI 3.0.2 Schema Object. The schema files that can be used here are found in gs://google-cloud-aiplatform/schema/dataset/metadata/.
modelReference	string	Optional. Reference to the public base model last used by the dataset. Only set for prompt datasets.
satisfiesPzi	boolean	Output only. Reserved for future use.
satisfiesPzs	boolean	Output only. Reserved for future use.
savedQueries	array	All SavedQueries belong to the Dataset will be returned in List/Get Dataset response. The annotation_specs field will not be populated except for UI cases which will only use annotation_spec_count. In CreateDataset request, a SavedQuery is created together if this field is set, up to one SavedQuery can be set in CreateDatasetRequest. The SavedQuery should not contain any AnnotationSpec.
updateTime	string (google-datetime)	Output only. Timestamp when this Dataset was last updated.

Methods

The following methods are available for this resource:

Name	Accessible by	Required Params	Optional Params	Description
get	select	datasetsId	readMask	Gets a Dataset.
list	select		pageSize, pageToken, orderBy, filter, readMask, parent	Lists Datasets in a Location.
create	insert		parent	Creates a Dataset.
patch	update	datasetsId	updateMask	Updates a Dataset.
delete	delete	datasetsId		Deletes a Dataset.
import	exec	projectsId, locationsId, datasetsId		Imports data into a Dataset.
search_data_items	exec	projectsId, locationsId, datasetsId	annotationsLimit, dataLabelingJob, annotationsFilter, fieldMask, pageSize, orderBy, pageToken, orderByAnnotation.orderBy, savedQuery, dataItemFilter, orderByAnnotation.savedQuery, annotationFilters, orderByDataItem	Searches DataItems in a Dataset.
export	exec	projectsId, locationsId, datasetsId		Exports data from a Dataset.

Parameters

Parameters can be passed in the WHERE clause of a query. Check the Methods section to see which parameters are required or optional for each operation.

Name	Datatype	Description
datasetsId	string
locationsId	string
projectsId	string
annotationFilters	string
annotationsFilter	string
annotationsLimit	integer (int32)
dataItemFilter	string
dataLabelingJob	string
fieldMask	string (google-fieldmask)
filter	string
orderBy	string
orderByAnnotation.orderBy	string
orderByAnnotation.savedQuery	string
orderByDataItem	string
pageSize	integer (int32)
pageToken	string
parent	string
readMask	string (google-fieldmask)
savedQuery	string
updateMask	string (google-fieldmask)

SELECT examples

get

Gets a Dataset.

SELECT
name,
createTime,
dataItemCount,
description,
displayName,
encryptionSpec,
etag,
labels,
metadata,
metadataArtifact,
metadataSchemaUri,
modelReference,
satisfiesPzi,
satisfiesPzs,
savedQueries,
updateTime
FROM google.aiplatform.datasets
WHERE datasetsId = '{{ datasetsId }}' -- required
AND readMask = '{{ readMask }}'
;

list

Lists Datasets in a Location.

SELECT
name,
createTime,
dataItemCount,
description,
displayName,
encryptionSpec,
etag,
labels,
metadata,
metadataArtifact,
metadataSchemaUri,
modelReference,
satisfiesPzi,
satisfiesPzs,
savedQueries,
updateTime
FROM google.aiplatform.datasets
WHERE pageSize = '{{ pageSize }}'
AND pageToken = '{{ pageToken }}'
AND orderBy = '{{ orderBy }}'
AND filter = '{{ filter }}'
AND readMask = '{{ readMask }}'
AND parent = '{{ parent }}'
;

INSERT examples

create

Creates a Dataset.

INSERT INTO google.aiplatform.datasets (
data__metadataSchemaUri,
data__modelReference,
data__labels,
data__metadata,
data__encryptionSpec,
data__displayName,
data__etag,
data__description,
data__savedQueries,
parent
)
SELECT 
'{{ metadataSchemaUri }}',
'{{ modelReference }}',
'{{ labels }}',
'{{ metadata }}',
'{{ encryptionSpec }}',
'{{ displayName }}',
'{{ etag }}',
'{{ description }}',
'{{ savedQueries }}',
'{{ parent }}'
RETURNING
name,
done,
error,
metadata,
response
;

Manifest

# Description fields are for documentation purposes
- name: datasets
props:
  - name: metadataSchemaUri
    value: "{{ metadataSchemaUri }}"
    description: |
      Required. Points to a YAML file stored on Google Cloud Storage describing additional information about the Dataset. The schema is defined as an OpenAPI 3.0.2 Schema Object. The schema files that can be used here are found in gs://google-cloud-aiplatform/schema/dataset/metadata/.
  - name: modelReference
    value: "{{ modelReference }}"
    description: |
      Optional. Reference to the public base model last used by the dataset. Only set for prompt datasets.
  - name: labels
    value: "{{ labels }}"
    description: |
      The labels with user-defined metadata to organize your Datasets. Label keys and values can be no longer than 64 characters (Unicode codepoints), can only contain lowercase letters, numeric characters, underscores and dashes. International characters are allowed. No more than 64 user labels can be associated with one Dataset (System labels are excluded). See https://goo.gl/xmQnxf for more information and examples of labels. System reserved label keys are prefixed with "aiplatform.googleapis.com/" and are immutable. Following system labels exist for each Dataset: * "aiplatform.googleapis.com/dataset_metadata_schema": output only, its value is the metadata_schema's title.
  - name: metadata
    value: "{{ metadata }}"
    description: |
      Required. Additional information about the Dataset.
  - name: encryptionSpec
    description: |
      Customer-managed encryption key spec for a Dataset. If set, this Dataset and all sub-resources of this Dataset will be secured by this key.
    value:
      kmsKeyName: "{{ kmsKeyName }}"
  - name: displayName
    value: "{{ displayName }}"
    description: |
      Required. The user-defined name of the Dataset. The name can be up to 128 characters long and can consist of any UTF-8 characters.
  - name: etag
    value: "{{ etag }}"
    description: |
      Used to perform consistent read-modify-write updates. If not set, a blind "overwrite" update happens.
  - name: description
    value: "{{ description }}"
    description: |
      The description of the Dataset.
  - name: savedQueries
    description: |
      All SavedQueries belong to the Dataset will be returned in List/Get Dataset response. The annotation_specs field will not be populated except for UI cases which will only use annotation_spec_count. In CreateDataset request, a SavedQuery is created together if this field is set, up to one SavedQuery can be set in CreateDatasetRequest. The SavedQuery should not contain any AnnotationSpec.
    value:
      - updateTime: "{{ updateTime }}"
        displayName: "{{ displayName }}"
        etag: "{{ etag }}"
        createTime: "{{ createTime }}"
        supportAutomlTraining: {{ supportAutomlTraining }}
        problemType: "{{ problemType }}"
        annotationSpecCount: {{ annotationSpecCount }}
        metadata: "{{ metadata }}"
        name: "{{ name }}"
        annotationFilter: "{{ annotationFilter }}"
  - name: parent
    value: "{{ parent }}"

UPDATE examples

patch

Updates a Dataset.

UPDATE google.aiplatform.datasets
SET 
data__metadataSchemaUri = '{{ metadataSchemaUri }}',
data__modelReference = '{{ modelReference }}',
data__labels = '{{ labels }}',
data__metadata = '{{ metadata }}',
data__encryptionSpec = '{{ encryptionSpec }}',
data__displayName = '{{ displayName }}',
data__etag = '{{ etag }}',
data__description = '{{ description }}',
data__savedQueries = '{{ savedQueries }}'
WHERE 
datasetsId = '{{ datasetsId }}' --required
AND updateMask = '{{ updateMask}}'
RETURNING
name,
createTime,
dataItemCount,
description,
displayName,
encryptionSpec,
etag,
labels,
metadata,
metadataArtifact,
metadataSchemaUri,
modelReference,
satisfiesPzi,
satisfiesPzs,
savedQueries,
updateTime;

DELETE examples

delete

Deletes a Dataset.

DELETE FROM google.aiplatform.datasets
WHERE datasetsId = '{{ datasetsId }}' --required
;

Lifecycle Methods

import

Imports data into a Dataset.

EXEC google.aiplatform.datasets.import 
@projectsId='{{ projectsId }}' --required, 
@locationsId='{{ locationsId }}' --required, 
@datasetsId='{{ datasetsId }}' --required 
@@json=
'{
"importConfigs": "{{ importConfigs }}"
}'
;

search_data_items

Searches DataItems in a Dataset.

EXEC google.aiplatform.datasets.search_data_items 
@projectsId='{{ projectsId }}' --required, 
@locationsId='{{ locationsId }}' --required, 
@datasetsId='{{ datasetsId }}' --required, 
@annotationsLimit='{{ annotationsLimit }}', 
@dataLabelingJob='{{ dataLabelingJob }}', 
@annotationsFilter='{{ annotationsFilter }}', 
@fieldMask='{{ fieldMask }}', 
@pageSize='{{ pageSize }}', 
@orderBy='{{ orderBy }}', 
@pageToken='{{ pageToken }}', 
@orderByAnnotation.orderBy='{{ orderByAnnotation.orderBy }}', 
@savedQuery='{{ savedQuery }}', 
@dataItemFilter='{{ dataItemFilter }}', 
@orderByAnnotation.savedQuery='{{ orderByAnnotation.savedQuery }}', 
@annotationFilters='{{ annotationFilters }}', 
@orderByDataItem='{{ orderByDataItem }}'
;

export

Exports data from a Dataset.

EXEC google.aiplatform.datasets.export 
@projectsId='{{ projectsId }}' --required, 
@locationsId='{{ locationsId }}' --required, 
@datasetsId='{{ datasetsId }}' --required 
@@json=
'{
"exportConfig": "{{ exportConfig }}"
}'
;