openapi: 3.0.3

info:
  title: Decodable Data Plane
  description: Decodable Data Plane API specification.
  version: v1alpha2

servers:
  - url: https://{account}.api.data.decodable.co/{version}
    description: Production API server for all data operations.
    variables:
      account:
        description: Decodable account name. Required.
        default: "replace-with-account"
      version:
        description: The API version. Required.
        default: v1alpha2

  - url: http://{account}.localhost{port}/{version}
    description: Local testing and development API server.
    variables:
      account:
        description: Decodable account name. Required.
        default: "replace-with-account"
      version:
        description: The API version. Required.
        default: v1alpha2
      port:
        description: |
          An optional port.

          This is strictly for testing. When used, a colon must be prepended.
        default: ""

tags:
  - name: Stream Operations
  - name: Preview Operations
  - name: Secret Operations
  - name: Connectivity Operations
  - name: Connection Operations
  - name: Pipeline Operations

security:
  - bearerAuth: [ ]

paths:

  # Stream APIs

  /streams/{id}/preview:
    parameters:
      - $ref: "#/components/parameters/ResourceId"
    get:
      operationId: GetStreamPreview
      summary: Get a preview of the data on the stream.
      description: |
        Requires a bearer token for data plane stream preview issued by the control plane API.
      tags:
        - Stream Operations
      responses:
        200:
          $ref: "#/components/responses/StreamPreviewResponse"
        403:
          $ref: "#/components/responses/ErrorForbiddenResponse"
        404:
          $ref: "#/components/responses/ErrorResourceDoesNotExistResponse"
        default:
          $ref: "#/components/responses/ErrorGenericResponse"

  /streams/{id}/clear:
    parameters:
      - $ref: "#/components/parameters/ResourceId"
    post:
      operationId: StreamClear
      summary: Delete all data on the stream.
      description: |
        Requires a bearer token for data plane stream clear issued by the control plane API.
      tags:
        - Stream Operations
      responses:
        200:
          description: Stream successfully cleared.
        403:
          $ref: "#/components/responses/ErrorForbiddenResponse"
        404:
          $ref: "#/components/responses/ErrorResourceDoesNotExistResponse"
        default:
          $ref: "#/components/responses/ErrorGenericResponse"

  # SQL Preview APIs

  /sql/preview:
    post:
      operationId: CreateSQLPreview
      summary: Create a SQL preview.
      description: |
        Requires a bearer token for starting a data plane preview issued by the control plane API.
      tags:
        - Preview Operations
      requestBody:
        $ref: "#/components/requestBodies/CreateSqlPreviewRequest"
      responses:
        201:
          $ref: "#/components/responses/CreatePreviewResponse"
        400:
          $ref: "#/components/responses/ErrorBadRequestResponse"
        403:
          $ref: "#/components/responses/ErrorForbiddenResponse"
        default:
          $ref: "#/components/responses/ErrorGenericResponse"

    get:
      operationId: GetSqlPreview
      parameters:
        - name: decodable-token
          description: An encoded token provided when a preview is created or fetched (if additional results from the preview are expected)
          in: header
          required: true
          schema:
            type: string
      summary: Get preview results by id.
      description: |
        Requires a bearer token for getting a data plane preview issued by the control plane API.
        Also requires the `decodable-token` header, which is provided by the data plane and signed to prevent tampering.
      tags:
        - Preview Operations
      responses:
        200:
          $ref: "#/components/responses/GetPreviewResponse"
        400:
          $ref: "#/components/responses/ErrorBadRequestResponse"
        404:
          $ref: "#/components/responses/ErrorResourceDoesNotExistResponse"
        410:
          $ref: "#/components/responses/ErrorGoneResponse"
        default:
          $ref: "#/components/responses/ErrorGenericResponse"

  # Secrets APIs

  /secrets/plaintext:
    post:
      operationId: WritePlaintextSecret
      parameters:
        - name: decodable-plaintext-base64
          description: |
            The base64 encoded plaintext value of the secret to be stored.
            Must not be empty.
          in: header
          required: true
          schema:
            type: string
      summary: Get preview results by id.
      description: |
        Requires a bearer token for writing a plaintext secret issued by the control plane API.
      tags:
        - Secret Operations
      requestBody:
        $ref: "#/components/requestBodies/WriteSecretPlaintextTokenRequest"
      responses:
        204:
          description: The secret value was written successfully
        400:
          $ref: "#/components/responses/ErrorBadRequestResponse"
        default:
          $ref: "#/components/responses/ErrorGenericResponse"

  # Connectivity Check APIs

  /connectivity-check:
    post:
      operationId: RunConnectivityCheck
      summary: Run a connectivity check.
      description: |
        Requires a bearer token for running a connectivity check issued by the control plane API.
      tags:
        - Connectivity Operations
      responses:
        204:
          description: The connectivity check was successful
        400:
          $ref: "#/components/responses/ErrorBadRequestResponse"
        default:
          $ref: "#/components/responses/ErrorGenericResponse"

  /connections/{id}/metrics:
    parameters:
      - $ref: "#/components/parameters/ResourceId"
    post:
      operationId: GetConnectionMetrics
      requestBody:
        $ref: "#/components/requestBodies/MetricsRequest"
      summary: Retrieve metrics for a given connection
      description: |
        Requires a bearer token for retrieving connection metrics issued by the control plane API.
      tags:
        - Connection Operations
      responses:
        200:
          $ref: "#/components/responses/GetMetricsResponse"
        400:
          $ref: "#/components/responses/ErrorBadRequestResponse"
        default:
          $ref: "#/components/responses/ErrorGenericResponse"

  # Pipeline APIs

  /pipelines/custom/{id}/logs:
    parameters:
      - $ref: "#/components/parameters/ResourceId"
    get:
      operationId: GetCustomPipelineLogs
      summary: Retrieve logs for a custom pipeline
      description: |
        Requires a bearer token for retrieving the logs from a custom pipeline issued by the control plane API.
        Will return a set of plaintext logs when using the `Accept: text/plain` header. If using
        `Accept: text/event-stream`, this will return a server-sent-event stream of logs.
        
        Note that the query parameters must be the exact same (and in the same order) as was
        used in the request for a token to the control plane.
      tags:
        - Pipeline Operations
      parameters:
        - $ref: "#/components/parameters/TaskIndex"
        - $ref: "#/components/parameters/NumLines"
        - $ref: "#/components/parameters/Follow"
      responses:
        200:
          $ref: "#/components/responses/GetLogsResponse"
        400:
          $ref: "#/components/responses/ErrorBadRequestResponse"
        default:
          $ref: "#/components/responses/ErrorGenericResponse"

  /pipelines/{id}/metrics:
    parameters:
      - $ref: "#/components/parameters/ResourceId"
    post:
      operationId: GetPipelineMetrics
      requestBody:
        $ref: "#/components/requestBodies/MetricsRequest"
      summary: Retrieve metrics for a given pipeline
      description: |
        Requires a bearer token for retrieving pipeline metrics issued by the control plane API.
      tags:
        - Pipeline Operations
      responses:
        200:
          $ref: "#/components/responses/GetMetricsResponse"
        400:
          $ref: "#/components/responses/ErrorBadRequestResponse"
        default:
          $ref: "#/components/responses/ErrorGenericResponse"

  /pipelines/{id}/flink-ui/**:
    parameters:
      - $ref: "#/components/parameters/ResourceId"
    get:
      operationId: ForwardFlinkUi
      summary: Send a request to the Flink REST API of a running custom pipeline
      description: |
        Requires a bearer token for Flink API access issued by the control plane API.
      tags:
        - Pipeline Operations
      responses:
        200:
          description: Proxied response by the Flink REST API
        400:
          $ref: "#/components/responses/ErrorBadRequestResponse"
        default:
          $ref: "#/components/responses/ErrorGenericResponse"

  # File APIs

  /files:
    post:
      operationId: UploadFile
      summary: Upload a file to Decodable.
      description: |
        Requires a bearer token for file upload issued by the control plane API.
      tags:
        - File Operations
      requestBody:
        content:
          multipart/form-data:
            schema:
              required:
                - type
                - request
                - file
              type: object
              properties:
                type:
                  type: string
                  enum:
                    - JOB_FILE
                    - CONFIG_FILE
                request:
                  type: object
                  properties:
                    names:
                      type: array
                      items:
                        type: string
                file:
                  type: string
                  format: binary
            encoding:
              type:
                contentType: application/json
              request:
                contentType: application/json
      responses:
        200:
          $ref: "#/components/schemas/File"
        201:
          $ref: "#/components/schemas/File"
        400:
          $ref: "#/components/responses/ErrorBadRequestResponse"
        403:
          $ref: "#/components/responses/ErrorForbiddenResponse"
        default:
          $ref: "#/components/responses/ErrorGenericResponse"

components:

  schemas:

    ResourceId:
      description: An opaque ID, created by Decodable, that uniquely identifies a resource.
      type: string
      example: abcd1234

    ConnectorType:
      description: Direction of the Decodable connector.
      type: string
      enum:
        - source
        - sink

    FieldKind:
      description: The type of the field.
      type: string
      enum:
        - physical
        - computed
        - metadata

    PhysicalField:
      description: A field which is physically present in the data.
      type: object
      properties:
        kind:
          $ref: "#/components/schemas/FieldKind"
        name:
          description: The name of the field.
          type: string
        type:
          description: The field's type.
          type: string

    ComputedField:
      description: A field which is computed at runtime.
      type: object
      properties:
        kind:
          $ref: "#/components/schemas/FieldKind"
        name:
          description: The name of the field.
          type: string
        expression:
          description: The expression defining the value taken by the field at runtime.

    MetadataField:
      description: A field which exposes metadata from the connector.
      type: object
      properties:
        kind:
          $ref: "#/components/schemas/FieldKind"
        name:
          description: The name of the field.
          type: string
        type:
          description: The field's type.
          type: string
        key:
          description: The key used to retrieve the metadata from the connector.
          type: string

    Metrics:
      description: A metrics response.
      type: object
      required:
        - sources
        - sinks
        - update_time
      properties:
        sources:
          type: object
          additionalProperties:
            type: object
            properties:
              bytes_in_total:
                type: number
                format: double
              bytes_in_per_second:
                type: number
                format: double
              records_in_total:
                type: number
                format: double
              records_in_per_second:
                type: number
                format: double
        sinks:
          type: object
          additionalProperties:
            type: object
            properties:
              bytes_out_total:
                type: number
                format: double
              bytes_out_per_second:
                type: number
                format: double
              records_out_total:
                type: number
                format: double
              records_out_per_second:
                type: number
                format: double
        additional:
          type: object
          additionalProperties:
            type: number
            format: double
        update_time:
          type: string
          format: date-time

    SchemaV2:
      description: A stream or connection schema in the V2 format.
      type: object
      properties:
        fields:
          description: Schema fields
          type: array
          items:
            oneOf:
              - $ref: "#/components/schemas/PhysicalField"
              - $ref: "#/components/schemas/ComputedField"
              - $ref: "#/components/schemas/MetadataField"
        watermarks:
          type: array
          items:
            description: The watermark defining the grace period for late-arriving data.
            type: object
            properties:
              name:
                description: The name of the field used in the watermark.
                type: string
              expression:
                description: The expression defining the watermark.
                type: string
        constraints:
          type: object
          properties:
            primary_key:
              description: The names of the fields used in the schema's primary key
              type: array
              items:
                description: The name of a field in the schema.
                type: string

    StreamPreview:
      description: A stream preview response.
      type: object
      required:
        - results
      properties:
        results:
          description: Result rows from preview.
          type: array
          items:
            description: Preview result row
            type: object
            required:
              - value
              - _timestamp
            properties:
              _timestamp:
                description: The processing time as a unix timestamp in milliseconds
                type: integer
              key:
                description: The record's primary key
                type: object
              value:
                description: The record value
                type: object

    SqlPreviewCreateRequest:
      description: A preview creation request.
      type: object
      required:
        - preview_id
        - rewritten_sql
        - stream_metadata
        - output_metadata
      properties:
        preview_id:
          description: The id for the preview to use
          type: string
        rewritten_sql:
          description: The sql for the preview to run.
          type: string
        stream_metadata:
          type: array
          items:
            type: object
            additionalProperties:
              $ref: "#/components/schemas/StreamMetadata"
        output_metadata:
          description: A descriptor of the preview output
          type: object
          $ref: "#/components/schemas/OutputMetadata"

    Preview:
      description: A preview response.
      type: object
      required:
        - id
      properties:
        id:
          $ref: "#/components/schemas/ResourceId"
        output_stream_type:
          type: string
          enum:
            - APPEND
            - CHANGE
        results:
          description: Result rows from preview.
          type: array
          items:
            description: Preview result row
            type: object
        next_token:
          description: A token used to fetch the next available results.
          type: string

    StreamMetadata:
      description: Stream context to allow the data plane to execute the requested operation.
      type: object
      properties:
        id:
          type: string
        name:
          type: string
        connector:
          type: string
        schema_fields:
          type: object
          description: An ordered map of field names to their respective field types
          additionalProperties:
            type: string
        ddl_properties:
          description: Properties to be used in Flink ddl when executing the preview
          additionalProperties:
            type: string
        stream_resource_properties:
          description: Decodable stream properties
          additionalProperties:
            type: string
        stream_type:
          type: string
          description: Decodable stream type
        watermark:
          type: string

    OutputMetadata:
      description: A descriptor of a preview output.
      type: object
      properties:
        output_stream_type:
          description: The stream type of the preview output
          type: string
        name:
          description: The name for the preview output
          type: string
        schema:
          $ref: "#/components/schemas/SchemaV2"
        ddl_properties:
          description: Properties to be used in Flink ddl when executing the preview
          additionalProperties:
            type: string

    File:
      description: Metadata for a file.
      type: object
      properties:
        sha:
          type: string
          description: The sha512 hash of the file
        type:
          type: string
          description: The type of the file
          enum:
            - JOB_FILE
            - CONFIG_FILE
        size:
          type: number
          description: The file size in bytes
        metadata:
          type: object
          description: Additional metadata for the file

    Error:
      description: |
        Information about an error that occurred.

        Note that most of the field values are not guaranteed to be stable and are not to be considered part of the
        API contract. In most cases, they are for informational purposes and to aid in reporting errors to Decodable.
      type: object
      required:
        - message
        - time
      properties:
        request_id:
          description: A unique request ID for support.
          type: string
        message:
          description: A human readable description of the error.
          type: string
        code:
          description: A code describing the exact source of the error.
          type: integer
        time:
          description: The time the error occurred, according to the service.
          type: string
          format: date-time
        details:
          description: Additional error-specific key/value pairs.
          type: object
          additionalProperties:
            type: string


  parameters:

    ResourceId:
      name: id
      description: An ID that uniquely identifies a resource.
      in: path
      required: true
      schema:
        type: string
        minLength: 1
        maxLength: 256
        pattern: "[a-zA-Z0-9]+(-[a-zA-Z0-9]+)*"
        example: abcd1234

    TaskIndex:
      description: An index for the task logs to query. 0 implies JobManager, 1+ are indices for TaskManagers
      name: task_index
      in: query
      schema:
        type: integer
        minimum: 0
        example: 0
        default: 0

    NumLines:
      description: The number of lines to return
      name: num_lines
      in: query
      schema:
        type: integer
        minimum: 1
        example: 20
        default: 20

    Follow:
      description: Whether the logging request should be made for following logs
      name: follow
      in: query
      schema:
        type: boolean
        default: false


  requestBodies:

    # Preview Requests

    CreateSqlPreviewRequest:
      description: A SQL preview creation request.
      required: true
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/SqlPreviewCreateRequest"

    # Secret Requests

    WriteSecretPlaintextTokenRequest:
      description: A request to write a plaintext secret with a given ID.
      required: true
      content:
        application/json:
          schema:
            type: object
            required:
              - id
            properties:
              id:
                $ref: "#/components/schemas/ResourceId"

    # Connectivity Requests

    ConnectivityCheckRequest:
      description: A request to run a connectivity check.
      required: true
      content:
        application/json:
          schema:
            type: object
            required:
              - connector
              - type
              - schema_v2
              - properties
            properties:
              connector:
                description: Decodable connector name.
                type: string
              type:
                $ref: "#/components/schemas/ConnectorType"
              schema_v2:
                $ref: "#/components/schemas/SchemaV2"
              properties:
                description: Properties allowing connectivity with the external resource.
                type: object
                additionalProperties:
                  type: string

    # Metrics Requests
    MetricsRequest:
      description: A request to retrieve metrics for a given job
      required: true
      content:
        application/json:
          schema:
            type: object
            required:
              - flinkVersion
            properties:
              flinkVersion:
                description: Which version of Flink the job is running on
                type: string


  responses:

    # Stream Responses

    StreamPreviewResponse:
      description: A stream preview response.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/StreamPreview"

    # File Responses

    UploadFileResponse:
      description: An upload file response.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/File"

    # Sql Preview Responses

    CreatePreviewResponse:
      description: A preview creation response.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/Preview"

    GetPreviewResponse:
      description: A preview get response.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/Preview"

    GetMetricsResponse:
      description: A metrics get response
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/Metrics"


    # Logs Responses

    GetLogsResponse:
      description: "A logs response"
      content:
        text/plain:
          schema:
            type: string
        text/event-stream:
          schema:
            type: string


    # Error Responses

    ErrorForbiddenResponse:
      description: "Access denied: request is not permitted."
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/Error"

    ErrorResourceDoesNotExistResponse:
      description: The requested resource does not exist.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/Error"

    ErrorResourceConflictResponse:
      description: A resource conflict has been detected.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/Error"

    ErrorBadRequestResponse:
      description: An invalid request was received.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/Error"

    ErrorTooManyRequestsResponse:
      description: Too many requests have been sent in a given amount of time.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/Error"

    ErrorGoneResponse:
      description: The requested resource no longer exists.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/Error"

    ErrorGenericResponse:
      description: An error has occurred.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/Error"


  securitySchemes:

    bearerAuth:
      type: http
      scheme: bearer
      bearerFormat: JWT
      description: |
        The data_plane_token issued by the control plane authorizing the request and providing necessary context.