From 54130efab4f668cf2c1b572275415b641e20f75f Mon Sep 17 00:00:00 2001 From: General Kroll Date: Sun, 26 Jan 2025 10:48:06 +1100 Subject: [PATCH] sqlite-mathematics-returns Summary: - Improved CICD doco. - Migrated duplicate functionality to `any-sdk`. - Restore `sqlite3` mathematical functions. - Added robot test `Google Buckets List With Date Logic Exemplifies Use of SQLite Math Functions`. --- docs/CICD.md | 5 +- go.mod | 4 +- go.sum | 4 +- .../internaldto/exec_payload.go | 42 ---- .../primitivegenerator/statement_analyzer.go | 7 +- test/python/flask/gcp/app.py | 6 +- .../flask/gcp/templates/buckets-list.json | 235 ++++++++++++++++++ .../stackql_mocked_from_cmd_line.robot | 40 +++ 8 files changed, 292 insertions(+), 51 deletions(-) delete mode 100644 internal/stackql/internal_data_transfer/internaldto/exec_payload.go create mode 100644 test/python/flask/gcp/templates/buckets-list.json diff --git a/docs/CICD.md b/docs/CICD.md index f1bb0723..a1070942 100644 --- a/docs/CICD.md +++ b/docs/CICD.md @@ -6,7 +6,6 @@ Summary: - At present, PR checks, build and test are all performed through [.github/workflows/build.yml](/.github/workflows/build.yml). - Releasing over various channels (website, homebrew, chocolatey...) is performed manually. -- ~~The strategic state is to split the functions: PR checks, build and test; into separate files, and migrate to use [goreleaser](https://goreleaser.com/).~~ - Docker Build and Push Jobs have scope for improvement. - These are currently based loosely on patterns described in: - https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners @@ -18,6 +17,10 @@ Summary: - **NOTE**: The QEMU build for linux/arm64 is **very slow**. On the order of 30 minutes. This is currently unavoidable. - **TODO**: Migrate linux/arm64 docker build to native once GHA supports this platform as a first class citizen. - **NOTE**. Be careful selecting from [the available github actions runners](https://github.com/actions/runner-images). Some runnner instances may be incompatible with assumed toolchain. +- Docker images: + - [production image at `stackql/stackql`](https://hub.docker.com/r/stackql/stackql/tags). + - [development (zero guarantees) image at `stackql/stackql-devel`](https://hub.docker.com/r/stackql/stackql-devel/tags). + ## Secrets diff --git a/go.mod b/go.mod index b62cae17..13831b69 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/jackc/pgx/v5 v5.0.4 github.com/lib/pq v1.10.4 github.com/magiconair/properties v1.8.6 - github.com/mattn/go-sqlite3 v1.0.2-stackql + github.com/mattn/go-sqlite3 v1.0.3-stackql github.com/olekukonko/tablewriter v0.0.0-20180130162743-b8a9be070da4 github.com/sirupsen/logrus v1.9.0 github.com/snowflakedb/gosnowflake v1.6.16 @@ -130,4 +130,4 @@ require ( replace github.com/chzyer/readline => github.com/stackql/readline v0.0.2-alpha05 -replace github.com/mattn/go-sqlite3 => github.com/stackql/stackql-go-sqlite3 v1.0.2-stackql +replace github.com/mattn/go-sqlite3 => github.com/stackql/stackql-go-sqlite3 v1.0.3-stackql diff --git a/go.sum b/go.sum index b87b67a2..2f98791a 100644 --- a/go.sum +++ b/go.sum @@ -490,8 +490,8 @@ github.com/stackql/psql-wire v0.1.1-alpha07 h1:LQWVUlx4Bougk6dztDNG5tmXxpIVeeTSs github.com/stackql/psql-wire v0.1.1-alpha07/go.mod h1:a44Wd8kDC3irFLpGutarKDBqhJ/aqXlj1aMzO5bVJYg= github.com/stackql/readline v0.0.2-alpha05 h1:ID4QzGdplFBsrSnTuz8pvKzWw96JbrJg8fsLry2UriU= github.com/stackql/readline v0.0.2-alpha05/go.mod h1:OFAYOdXk/X4+5GYiDXFfaGrk+bCN6Qv0SYY5HNzD2E0= -github.com/stackql/stackql-go-sqlite3 v1.0.2-stackql h1:HyctYfBzMth/fIs2Y+9c/SICsCvR/DEnzcI7VZgwGZo= -github.com/stackql/stackql-go-sqlite3 v1.0.2-stackql/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/stackql/stackql-go-sqlite3 v1.0.3-stackql h1:j0yt6T5thZuz5+HIr81PXz2AClAtCko0vzr5tm8C1g8= +github.com/stackql/stackql-go-sqlite3 v1.0.3-stackql/go.mod h1:HemqCrcMK2xyhMMMt6oZ7ERDtoSmyyDsw5LBBcTZ+Rk= github.com/stackql/stackql-parser v0.0.14-alpha04 h1:CmZUDWf2jBbU+Zu3sdiuM5CwUBFGN/IFjzXPcu46xjs= github.com/stackql/stackql-parser v0.0.14-alpha04/go.mod h1:iyB47SvRS+Fvpn7joF7mHAkeiWSq83TbUhglRmLzPLQ= github.com/stackql/stackql-provider-registry v0.0.1-rc06 h1:MgroWOr0bSqjSTDGnXB0UoZGFXpW3SRtN0EFkzB8Rpo= diff --git a/internal/stackql/internal_data_transfer/internaldto/exec_payload.go b/internal/stackql/internal_data_transfer/internaldto/exec_payload.go deleted file mode 100644 index 5ab50387..00000000 --- a/internal/stackql/internal_data_transfer/internaldto/exec_payload.go +++ /dev/null @@ -1,42 +0,0 @@ -package internaldto - -var ( - _ ExecPayload = &standardExecPayload{} -) - -type ExecPayload interface { - GetHeader() map[string][]string - GetPayload() []byte - GetPayloadMap() map[string]interface{} - SetHeaderKV(k string, v []string) -} - -func NewExecPayload(payload []byte, header map[string][]string, payloadMap map[string]interface{}) ExecPayload { - return &standardExecPayload{ - payload: payload, - header: header, - payloadMap: payloadMap, - } -} - -type standardExecPayload struct { - payload []byte - header map[string][]string - payloadMap map[string]interface{} -} - -func (ep *standardExecPayload) SetHeaderKV(k string, v []string) { - ep.header[k] = v -} - -func (ep *standardExecPayload) GetPayload() []byte { - return ep.payload -} - -func (ep *standardExecPayload) GetHeader() map[string][]string { - return ep.header -} - -func (ep *standardExecPayload) GetPayloadMap() map[string]interface{} { - return ep.payloadMap -} diff --git a/internal/stackql/primitivegenerator/statement_analyzer.go b/internal/stackql/primitivegenerator/statement_analyzer.go index 71354ac1..843290de 100644 --- a/internal/stackql/primitivegenerator/statement_analyzer.go +++ b/internal/stackql/primitivegenerator/statement_analyzer.go @@ -10,6 +10,7 @@ import ( "github.com/stackql/any-sdk/anysdk" "github.com/stackql/any-sdk/pkg/constants" "github.com/stackql/any-sdk/pkg/dto" + anysdk_internaldto "github.com/stackql/any-sdk/pkg/internaldto" "github.com/stackql/any-sdk/pkg/logging" "github.com/stackql/stackql/internal/stackql/astformat" "github.com/stackql/stackql/internal/stackql/astindirect" @@ -599,7 +600,7 @@ func (pb *standardPrimitiveGenerator) AnalyzeUnaryExec( if err != nil && reqExists { return nil, err } - var execPayload internaldto.ExecPayload + var execPayload anysdk_internaldto.ExecPayload if node.OptExecPayload != nil { mediaType := "application/json" //nolint:goconst // acceptable for now if reqExists && req.GetBodyMediaType() != "" { @@ -711,7 +712,7 @@ func (pb *standardPrimitiveGenerator) sanitiseMediaType(mediaType string) string func (pb *standardPrimitiveGenerator) parseExecPayload( node *sqlparser.ExecVarDef, payloadType string, -) (internaldto.ExecPayload, error) { +) (anysdk_internaldto.ExecPayload, error) { var b []byte m := make(map[string][]string) var pm map[string]interface{} @@ -732,7 +733,7 @@ func (pb *standardPrimitiveGenerator) parseExecPayload( default: return nil, fmt.Errorf("payload map of declared type = '%T' not allowed", payloadType) } - return internaldto.NewExecPayload( + return anysdk_internaldto.NewExecPayload( b, m, pm, diff --git a/test/python/flask/gcp/app.py b/test/python/flask/gcp/app.py index 44c55b40..9fff382f 100644 --- a/test/python/flask/gcp/app.py +++ b/test/python/flask/gcp/app.py @@ -12,7 +12,11 @@ def log_request_info(): logger.info(f"Request: {request.method} {request.path} - Query: {request.args}") - +@app.route('/b', methods=['GET']) +def v1_storage_buckets_list(): + if request.args.get('project') == 'stackql-demo': + return render_template('buckets-list.json'), 200, {'Content-Type': 'application/json'} + return '{"msg": "Project Not Found"}', 404, {'Content-Type': 'application/json'} @app.route('/v1/projects/testing-project-three/locations/global/keyRings/testing-three/cryptoKeys', methods=['GET']) def v1_projects_testing_project_three_locations_global_keyRings_testing_three_cryptoKeys(): diff --git a/test/python/flask/gcp/templates/buckets-list.json b/test/python/flask/gcp/templates/buckets-list.json new file mode 100644 index 00000000..da24b6b8 --- /dev/null +++ b/test/python/flask/gcp/templates/buckets-list.json @@ -0,0 +1,235 @@ +{ + "kind": "storage#buckets", + "items": [ + { + "kind": "storage#bucket", + "selfLink": "https://www.googleapis.com/storage/v1/b/demo-app-bucket1", + "id": "demo-app-bucket1", + "name": "demo-app-bucket1", + "projectNumber": "0000000003", + "generation": "0000000000001", + "metageneration": "1", + "location": "US", + "storageClass": "STANDARD", + "etag": "CAE=", + "timeCreated": "2023-02-17T05:33:56.248Z", + "updated": "2023-02-17T05:33:56.248Z", + "labels": { + "app": "stackql-demo-okta-app" + }, + "softDeletePolicy": { + "retentionDurationSeconds": "604800", + "effectiveTime": "2024-03-01T08:00:00.000Z" + }, + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": true, + "lockedTime": "2023-05-18T05:33:56.248Z" + }, + "uniformBucketLevelAccess": { + "enabled": true, + "lockedTime": "2023-05-18T05:33:56.248Z" + }, + "publicAccessPrevention": "enforced" + }, + "locationType": "multi-region", + "rpo": "DEFAULT" + }, + { + "kind": "storage#bucket", + "selfLink": "https://www.googleapis.com/storage/v1/b/demo-app-bucket2", + "id": "demo-app-bucket2", + "name": "demo-app-bucket2", + "projectNumber": "0000000003", + "generation": "0000000000001", + "metageneration": "1", + "location": "US", + "storageClass": "STANDARD", + "etag": "CAE=", + "timeCreated": "2023-02-17T05:34:26.958Z", + "updated": "2023-02-17T05:34:26.958Z", + "labels": { + "app": "stackql-demo-okta-app" + }, + "softDeletePolicy": { + "retentionDurationSeconds": "604800", + "effectiveTime": "2024-03-01T08:00:00.000Z" + }, + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": true, + "lockedTime": "2023-05-18T05:34:26.958Z" + }, + "uniformBucketLevelAccess": { + "enabled": true, + "lockedTime": "2023-05-18T05:34:26.958Z" + }, + "publicAccessPrevention": "enforced" + }, + "locationType": "multi-region", + "rpo": "DEFAULT" + }, + { + "kind": "storage#bucket", + "selfLink": "https://www.googleapis.com/storage/v1/b/stackql-demo-bucket", + "id": "stackql-demo-bucket", + "name": "stackql-demo-bucket", + "projectNumber": "0000000003", + "generation": "0000000000001", + "metageneration": "1", + "location": "US", + "storageClass": "STANDARD", + "etag": "CAE=", + "timeCreated": "2022-02-09T04:39:09.058Z", + "updated": "2022-02-09T04:39:09.058Z", + "softDeletePolicy": { + "retentionDurationSeconds": "604800", + "effectiveTime": "2024-03-01T08:00:00.000Z" + }, + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "uniformBucketLevelAccess": { + "enabled": false + }, + "publicAccessPrevention": "inherited" + }, + "locationType": "multi-region", + "rpo": "DEFAULT" + }, + { + "kind": "storage#bucket", + "selfLink": "https://www.googleapis.com/storage/v1/b/stackql-demo-src-bucket", + "id": "stackql-demo-src-bucket", + "name": "stackql-demo-src-bucket", + "projectNumber": "0000000003", + "generation": "0000000000001", + "metageneration": "3", + "location": "US", + "storageClass": "STANDARD", + "etag": "CAM=", + "timeCreated": "2022-02-08T23:23:47.208Z", + "updated": "2023-03-27T03:06:27.204Z", + "softDeletePolicy": { + "retentionDurationSeconds": "604800", + "effectiveTime": "2024-03-01T08:00:00.000Z" + }, + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "uniformBucketLevelAccess": { + "enabled": false + }, + "publicAccessPrevention": "inherited" + }, + "locationType": "multi-region", + "rpo": "DEFAULT" + }, + { + "kind": "storage#bucket", + "selfLink": "https://www.googleapis.com/storage/v1/b/stackql-demo.appspot.com", + "id": "stackql-demo.appspot.com", + "name": "stackql-demo.appspot.com", + "projectNumber": "0000000003", + "generation": "0000000000001", + "metageneration": "1", + "location": "US", + "storageClass": "STANDARD", + "etag": "CAE=", + "timeCreated": "2023-02-26T08:35:40.061Z", + "updated": "2023-02-26T08:35:40.061Z", + "softDeletePolicy": { + "retentionDurationSeconds": "604800", + "effectiveTime": "2024-03-01T08:00:00.000Z" + }, + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "uniformBucketLevelAccess": { + "enabled": false + }, + "publicAccessPrevention": "inherited" + }, + "locationType": "multi-region", + "rpo": "DEFAULT" + }, + { + "kind": "storage#bucket", + "selfLink": "https://www.googleapis.com/storage/v1/b/stackql-encrypted-bucket-1", + "id": "stackql-encrypted-bucket-1", + "name": "stackql-encrypted-bucket-1", + "projectNumber": "0000000003", + "generation": "0000000000001", + "metageneration": "1", + "location": "US", + "storageClass": "STANDARD", + "etag": "CAE=", + "timeCreated": "2023-02-28T03:18:33.043Z", + "updated": "2023-02-28T03:18:33.043Z", + "encryption": { + "defaultKmsKeyName": "projects/stackql-demo/locations/us/keyRings/stackql-us/cryptoKeys/stackql-us-demo-key" + }, + "softDeletePolicy": { + "retentionDurationSeconds": "604800", + "effectiveTime": "2024-03-01T08:00:00.000Z" + }, + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": true, + "lockedTime": "2023-05-29T03:18:33.043Z" + }, + "uniformBucketLevelAccess": { + "enabled": true, + "lockedTime": "2023-05-29T03:18:33.043Z" + }, + "publicAccessPrevention": "enforced" + }, + "locationType": "multi-region", + "rpo": "DEFAULT" + }, + { + "kind": "storage#bucket", + "selfLink": "https://www.googleapis.com/storage/v1/b/staging.stackql-demo.appspot.com", + "id": "staging.stackql-demo.appspot.com", + "name": "staging.stackql-demo.appspot.com", + "projectNumber": "0000000003", + "generation": "0000000000001", + "metageneration": "1", + "location": "US", + "storageClass": "STANDARD", + "etag": "CAE=", + "timeCreated": "2023-02-26T08:35:40.223Z", + "updated": "2023-02-26T08:35:40.223Z", + "lifecycle": { + "rule": [ + { + "action": { + "type": "Delete" + }, + "condition": { + "age": 15 + } + } + ] + }, + "softDeletePolicy": { + "retentionDurationSeconds": "604800", + "effectiveTime": "2024-03-01T08:00:00.000Z" + }, + "iamConfiguration": { + "bucketPolicyOnly": { + "enabled": false + }, + "uniformBucketLevelAccess": { + "enabled": false + }, + "publicAccessPrevention": "inherited" + }, + "locationType": "multi-region", + "rpo": "DEFAULT" + } + ] +} \ No newline at end of file diff --git a/test/robot/functional/stackql_mocked_from_cmd_line.robot b/test/robot/functional/stackql_mocked_from_cmd_line.robot index ba3a10fe..0943cd99 100644 --- a/test/robot/functional/stackql_mocked_from_cmd_line.robot +++ b/test/robot/functional/stackql_mocked_from_cmd_line.robot @@ -7425,3 +7425,43 @@ View Tuple Replacement Working As Exemplified by AWS EC2 Instances List and Deta ... stdout=${CURDIR}/tmp/View-Tuple-Replacement-Working-As-Exemplified-by-AWS-EC2-Instances-List-and-Detail.tmp ... stderr=${CURDIR}/tmp/View-Tuple-Replacement-Working-As-Exemplified-by-AWS-EC2-Instances-List-and-Detail-stderr.tmp ... repeat_count=20 + +Google Buckets List With Date Logic Exemplifies Use of SQLite Math Functions + Pass Execution If "${SQL_BACKEND}" == "postgres_tcp" This is a valid case where the test is targetted at SQLite only + ${inputStr} = Catenate + ... SELECT name, timeCreated, floor(julianday('2025-01-27')-julianday(timeCreated)) as days_since_ceiling + ... FROM google.storage.buckets + ... WHERE project = 'stackql-demo' + ... order by name desc + ... ; + ${outputStr} = Catenate SEPARATOR=\n + ... |----------------------------------|--------------------------|--------------------| + ... |${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}name${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}timeCreated${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}days_since_ceiling${SPACE}| + ... |----------------------------------|--------------------------|--------------------| + ... |${SPACE}staging.stackql-demo.appspot.com${SPACE}|${SPACE}2023-02-26T08:35:40.223Z${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}700${SPACE}| + ... |----------------------------------|--------------------------|--------------------| + ... |${SPACE}stackql-encrypted-bucket-1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}2023-02-28T03:18:33.043Z${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}698${SPACE}| + ... |----------------------------------|--------------------------|--------------------| + ... |${SPACE}stackql-demo.appspot.com${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}2023-02-26T08:35:40.061Z${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}700${SPACE}| + ... |----------------------------------|--------------------------|--------------------| + ... |${SPACE}stackql-demo-src-bucket${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}2022-02-08T23:23:47.208Z${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}1083${SPACE}| + ... |----------------------------------|--------------------------|--------------------| + ... |${SPACE}stackql-demo-bucket${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}2022-02-09T04:39:09.058Z${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}1082${SPACE}| + ... |----------------------------------|--------------------------|--------------------| + ... |${SPACE}demo-app-bucket2${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}2023-02-17T05:34:26.958Z${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}709${SPACE}| + ... |----------------------------------|--------------------------|--------------------| + ... |${SPACE}demo-app-bucket1${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}|${SPACE}2023-02-17T05:33:56.248Z${SPACE}|${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}${SPACE}709${SPACE}| + ... |----------------------------------|--------------------------|--------------------| + Should Stackql Exec Inline Equal Both Streams + ... ${STACKQL_EXE} + ... ${OKTA_SECRET_STR} + ... ${GITHUB_SECRET_STR} + ... ${K8S_SECRET_STR} + ... ${REGISTRY_NO_VERIFY_CFG_STR} + ... ${AUTH_CFG_STR} + ... ${SQL_BACKEND_CFG_STR_CANONICAL} + ... ${inputStr} + ... ${outputStr} + ... ${EMPTY} + ... stdout=${CURDIR}/tmp/Google-Buckets-List-With-Date-Logic-Exemplifies-Use-of-SQLite-Math-Functions.tmp + ... stderr=${CURDIR}/tmp/Google-Buckets-List-With-Date-Logic-Exemplifies-Use-of-SQLite-Math-Functions-stderr.tmp