diff --git a/.github/workflows/build-gpdb.yml b/.github/workflows/build-gpdb.yml index 828f194a415..b2641a8fb2c 100644 --- a/.github/workflows/build-gpdb.yml +++ b/.github/workflows/build-gpdb.yml @@ -156,6 +156,7 @@ jobs: "gpcontrib/gp_inject_fault:installcheck", "gpcontrib/gp_internal_tools:installcheck", "gpcontrib/gp_legacy_string_agg:installcheck", + "gpcontrib/gp_log_tools:installcheck", "gpcontrib/gp_percentile_agg:installcheck", "gpcontrib/gp_sparse_vector:installcheck", "gpcontrib/gp_subtransaction_overflow:installcheck", diff --git a/gpAux/gpdemo/demo_cluster.sh b/gpAux/gpdemo/demo_cluster.sh index 73a85ade494..becded89321 100755 --- a/gpAux/gpdemo/demo_cluster.sh +++ b/gpAux/gpdemo/demo_cluster.sh @@ -346,6 +346,7 @@ if [ "${ENABLE_COPY}" == "true" ]; then # Turn on COPY for cluster ycmdb.yc_allow_copy_to_program=on ycmdb.yc_allow_copy_from_file=on + ycmdb.yc_allow_copy_from_logs=on ycmdb.yc_allow_copy_to_file=on EOF diff --git a/gpcontrib/Makefile b/gpcontrib/Makefile index d9d6a26b74f..7430bb69449 100644 --- a/gpcontrib/Makefile +++ b/gpcontrib/Makefile @@ -21,6 +21,7 @@ ifeq "$(enable_debug_extensions)" "yes" gp_inject_fault \ gp_replica_check \ gp_legacy_string_agg \ + gp_log_tools \ gp_array_agg \ gp_percentile_agg \ gp_error_handling \ @@ -37,6 +38,7 @@ else gp_parser \ jsonlog-gpdb \ gp_legacy_string_agg \ + gp_log_tools \ gp_array_agg \ gp_percentile_agg \ gp_error_handling \ @@ -122,6 +124,7 @@ distclean: installcheck: $(MAKE) -C gp_internal_tools installcheck + $(MAKE) -C gp_log_tools installcheck $(MAKE) -C gp_array_agg installcheck if [ "$(enable_mapreduce)" = "yes" ]; then \ $(MAKE) -C gpmapreduce installcheck; \ diff --git a/gpcontrib/gp_log_tools/.gitignore b/gpcontrib/gp_log_tools/.gitignore new file mode 100644 index 00000000000..9fd0a070502 --- /dev/null +++ b/gpcontrib/gp_log_tools/.gitignore @@ -0,0 +1,3 @@ +results/* +regression.diffs +regression.out \ No newline at end of file diff --git a/gpcontrib/gp_log_tools/Makefile b/gpcontrib/gp_log_tools/Makefile new file mode 100644 index 00000000000..72da68f5e6e --- /dev/null +++ b/gpcontrib/gp_log_tools/Makefile @@ -0,0 +1,16 @@ +EXTENSION = gp_log_tools +DATA = gp_log_tools--0.0.1.sql + +REGRESS = gp_log_tools +REGRESS_OPTS += --init-file=$(top_builddir)/src/test/regress/init_file + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/gp_log_tools +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif \ No newline at end of file diff --git a/gpcontrib/gp_log_tools/expected/gp_log_tools.out b/gpcontrib/gp_log_tools/expected/gp_log_tools.out new file mode 100644 index 00000000000..3d3f17bbb68 --- /dev/null +++ b/gpcontrib/gp_log_tools/expected/gp_log_tools.out @@ -0,0 +1,25 @@ +CREATE EXTENSION gp_log_tools; +SHOW ycmdb.yc_allow_copy_from_logs; + ycmdb.yc_allow_copy_from_logs +------------------------------- + on +(1 row) + +SELECT count(*) != 0 as is_logs_selected FROM gp_master_logs; + is_logs_selected +------------------ + t +(1 row) + +CREATE USER test_user1; +GRANT SELECT ON gp_master_logs TO PUBLIC; +SET SESSION AUTHORIZATION test_user1; +SELECT count(*) != 0 as is_logs_selected FROM gp_master_logs; + is_logs_selected +------------------ + t +(1 row) + +\c - +DROP USER test_user1; +DROP EXTENSION gp_log_tools; diff --git a/gpcontrib/gp_log_tools/gp_log_tools--0.0.1.sql b/gpcontrib/gp_log_tools/gp_log_tools--0.0.1.sql new file mode 100644 index 00000000000..b7fe6d8443c --- /dev/null +++ b/gpcontrib/gp_log_tools/gp_log_tools--0.0.1.sql @@ -0,0 +1,85 @@ +\echo Use "CREATE EXTENSION gp_log_tools" to load this file. \quit + +CREATE OR REPLACE FUNCTION get_logs() RETURNS TABLE( + logtime timestamp with time zone, + loguser text, + logdatabase text, + logpid text, + logthread text, + loghost text, + logport text, + logsessiontime timestamp with time zone, + logtransaction int, + logsession text, + logcmdcount text, + logsegment text, + logslice text, + logdistxact text, + loglocalxact text, + logsubxact text, + logseverity text, + logstate text, + logmessage text, + logdetail text, + loghint text, + logquery text, + logquerypos int, + logcontext text, + logdebug text, + logcursorpos int, + logfunction text, + logfile text, + logline int, + logstack text +) AS +$BODY$ +DECLARE + r RECORD; +BEGIN + CREATE TEMP TABLE my_logs( + logtime timestamp with time zone, + loguser text, + logdatabase text, + logpid text, + logthread text, + loghost text, + logport text, + logsessiontime timestamp with time zone, + logtransaction int, + logsession text, + logcmdcount text, + logsegment text, + logslice text, + logdistxact text, + loglocalxact text, + logsubxact text, + logseverity text, + logstate text, + logmessage text, + logdetail text, + loghint text, + logquery text, + logquerypos int, + logcontext text, + logdebug text, + logcursorpos int, + logfunction text, + logfile text, + logline int, + logstack text + ) DISTRIBUTED RANDOMLY; + + FOR r IN + SELECT pg_ls_dir AS log_file FROM pg_ls_dir('pg_log') WHERE pg_ls_dir LIKE '%.csv' OR pg_ls_dir LIKE '%.log' + LOOP + EXECUTE $$COPY my_logs FROM 'pg_log/$$ || r.log_file || $$' CSV DELIMITER ',' QUOTE '"'$$; + END LOOP; + RETURN QUERY SELECT * FROM my_logs; + DROP TABLE my_logs; +END +$BODY$ +LANGUAGE plpgsql +SECURITY DEFINER; + +CREATE VIEW gp_master_logs AS +SELECT * FROM get_logs(); diff --git a/gpcontrib/gp_log_tools/gp_log_tools.control b/gpcontrib/gp_log_tools/gp_log_tools.control new file mode 100644 index 00000000000..26fefd81485 --- /dev/null +++ b/gpcontrib/gp_log_tools/gp_log_tools.control @@ -0,0 +1,3 @@ +comment = '' +default_version = '0.0.1' +relocatable = true \ No newline at end of file diff --git a/gpcontrib/gp_log_tools/sql/gp_log_tools.sql b/gpcontrib/gp_log_tools/sql/gp_log_tools.sql new file mode 100644 index 00000000000..5c074599a7e --- /dev/null +++ b/gpcontrib/gp_log_tools/sql/gp_log_tools.sql @@ -0,0 +1,19 @@ +CREATE EXTENSION gp_log_tools; + +SHOW ycmdb.yc_allow_copy_from_logs; + +SELECT count(*) != 0 as is_logs_selected FROM gp_master_logs; + +CREATE USER test_user1; + +GRANT SELECT ON gp_master_logs TO PUBLIC; + +SET SESSION AUTHORIZATION test_user1; + +SELECT count(*) != 0 as is_logs_selected FROM gp_master_logs; + +\c - + +DROP USER test_user1; + +DROP EXTENSION gp_log_tools; diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 31332048187..2e317ced1fa 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -42,6 +42,7 @@ #include "optimizer/clauses.h" #include "optimizer/planner.h" #include "parser/parse_relation.h" +#include "postmaster/syslogger.h" #include "rewrite/rewriteHandler.h" #include "storage/fd.h" #include "tcop/tcopprot.h" @@ -398,6 +399,8 @@ typedef struct bool yc_allow_copy_to_program; bool yc_allow_copy_to_file; bool yc_allow_copy_from_file; +bool yc_allow_copy_from_logs; + /* * Send copy start/stop messages for frontend copies. These have changed @@ -939,6 +942,45 @@ CopyLoadRawBuf(CopyState cstate) return (inbytes > 0); } +bool file_is_logs(const char *filepath) +{ + char* filename; + char* abs_log_path; + bool is_logs = false; + + if (filepath == NULL || DataDir == NULL) + return false; + + filename = strdup(filepath); + + canonicalize_path(filename); + + if (path_contains_parent_reference(filename)) { + free(filename); + return false; + } + + if (is_absolute_path(filename)) { + if (is_absolute_path(Log_directory)) { + abs_log_path = strdup(Log_directory); + } else { + abs_log_path = palloc(strlen(DataDir) + strlen(Log_directory) + 2); + join_path_components(abs_log_path, DataDir, Log_directory); + } + + is_logs = path_is_prefix_of_path(abs_log_path, filename); + pfree(abs_log_path); + } else if (path_is_relative_and_below_cwd(filename)) { + is_logs = path_is_prefix_of_path(Log_directory, filename); + } else { + is_logs = false; + } + + free(filename); + + return is_logs; +} + /* * DoCopy executes the SQL COPY statement @@ -971,6 +1013,7 @@ DoCopy(const CopyStmt *stmt, const char *queryString, uint64 *processed) AclMode required_access = (is_from ? ACL_INSERT : ACL_SELECT); TupleDesc tupDesc; List *options; + bool copy_from_logs; glob_cstate = NULL; glob_copystmt = (CopyStmt *) stmt; @@ -997,8 +1040,10 @@ DoCopy(const CopyStmt *stmt, const char *queryString, uint64 *processed) } } else { if (is_from) { + copy_from_logs = file_is_logs(stmt->filename); + // this is copy from file. This only could legitimately happen in initdb - if (!(superuser() && yc_allow_copy_from_file)) { + if (!(superuser() && (yc_allow_copy_from_file || (copy_from_logs && yc_allow_copy_from_logs)))) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("forbidden to COPY from file in Yandex Cloud"), diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index c22b5e0f80e..681cd2f849d 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -1644,6 +1644,15 @@ static struct config_bool ConfigureNamesBool[] = NULL, NULL, NULL }, + { + {"ycmdb.yc_allow_copy_from_logs", PGC_SUSET, DEVELOPER_OPTIONS, + gettext_noop("Whether to enable COPY from logs in Yandex Cloud"), + }, + &yc_allow_copy_from_logs, + true, + NULL, NULL, NULL + }, + /* End-of-list marker */ { {NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL diff --git a/src/include/commands/copy.h b/src/include/commands/copy.h index aa029e3283d..360db5edfff 100644 --- a/src/include/commands/copy.h +++ b/src/include/commands/copy.h @@ -342,5 +342,6 @@ typedef struct GpDistributionData extern PGDLLIMPORT bool yc_allow_copy_to_program; extern PGDLLIMPORT bool yc_allow_copy_to_file; extern PGDLLIMPORT bool yc_allow_copy_from_file; +extern PGDLLIMPORT bool yc_allow_copy_from_logs; #endif /* COPY_H */ diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index 45adb3a496f..d678e461b7b 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -563,3 +563,4 @@ "ycmdb.yc_allow_copy_to_program", "ycmdb.yc_allow_copy_to_file", "ycmdb.yc_allow_copy_from_file", + "ycmdb.yc_allow_copy_from_logs", diff --git a/src/test/regress/GNUmakefile b/src/test/regress/GNUmakefile index 86b5479a1c1..f4154a6a6fb 100644 --- a/src/test/regress/GNUmakefile +++ b/src/test/regress/GNUmakefile @@ -165,6 +165,7 @@ submake-contrib-dummy_seclabel: .PHONY: hook-setup hook-setup: gpconfig -c ycmdb.yc_allow_copy_from_file -v true + gpconfig -c ycmdb.yc_allow_copy_from_logs -v true gpconfig -c ycmdb.yc_allow_copy_to_program -v true gpconfig -c ycmdb.yc_allow_copy_to_file -v true