Allow copy from logs

.github/workflows/build-gpdb.yml

@@ -156,6 +156,7 @@ jobs:
                                "gpcontrib/gp_inject_fault:installcheck",
                                "gpcontrib/gp_internal_tools:installcheck",
                                "gpcontrib/gp_legacy_string_agg:installcheck",
+                               "gpcontrib/gp_log_tools:installcheck",
                                "gpcontrib/gp_percentile_agg:installcheck",
                                "gpcontrib/gp_sparse_vector:installcheck",
                                "gpcontrib/gp_subtransaction_overflow:installcheck",

gpAux/gpdemo/demo_cluster.sh

@@ -346,6 +346,7 @@ if [ "${ENABLE_COPY}" == "true" ]; then
 		# Turn on COPY for cluster
 		ycmdb.yc_allow_copy_to_program=on
 		ycmdb.yc_allow_copy_from_file=on
+		ycmdb.yc_allow_copy_from_logs=on
 		ycmdb.yc_allow_copy_to_file=on
 
 	EOF

gpcontrib/Makefile

@@ -21,6 +21,7 @@ ifeq "$(enable_debug_extensions)" "yes"
                gp_inject_fault \
                gp_replica_check \
                gp_legacy_string_agg \
+			   gp_log_tools \
                gp_array_agg \
                gp_percentile_agg \
                gp_error_handling \
@@ -37,6 +38,7 @@ else
                gp_parser \
                jsonlog-gpdb \
                gp_legacy_string_agg \
+			   gp_log_tools \
                gp_array_agg \
                gp_percentile_agg \
                gp_error_handling \
@@ -122,6 +124,7 @@ distclean:
 
 installcheck:
 	$(MAKE) -C gp_internal_tools installcheck
+	$(MAKE) -C gp_log_tools installcheck
 	$(MAKE) -C gp_array_agg installcheck
 	if [ "$(enable_mapreduce)" = "yes" ]; then \
 		$(MAKE) -C gpmapreduce installcheck; \

gpcontrib/gp_log_tools/.gitignore

@@ -0,0 +1,3 @@
+results/*
+regression.diffs
+regression.out

gpcontrib/gp_log_tools/Makefile

@@ -0,0 +1,16 @@
+EXTENSION = gp_log_tools
+DATA = gp_log_tools--0.0.1.sql
+
+REGRESS = gp_log_tools
+REGRESS_OPTS += --init-file=$(top_builddir)/src/test/regress/init_file
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = contrib/gp_log_tools
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif

gpcontrib/gp_log_tools/expected/gp_log_tools.out

@@ -0,0 +1,25 @@
+CREATE EXTENSION gp_log_tools;
+SHOW ycmdb.yc_allow_copy_from_logs;
+ ycmdb.yc_allow_copy_from_logs 
+-------------------------------
+ on
+(1 row)
+
+SELECT count(*) != 0 as is_logs_selected FROM gp_master_logs;
+ is_logs_selected 
+------------------
+ t
+(1 row)
+
+CREATE USER test_user1;
+GRANT SELECT ON gp_master_logs TO PUBLIC;
+SET SESSION AUTHORIZATION test_user1;
+SELECT count(*) != 0 as is_logs_selected FROM gp_master_logs;
+ is_logs_selected 
+------------------
+ t
+(1 row)
+
+\c -
+DROP USER test_user1;
+DROP EXTENSION gp_log_tools;

gpcontrib/gp_log_tools/gp_log_tools--0.0.1.sql

@@ -0,0 +1,85 @@
+\echo Use "CREATE EXTENSION gp_log_tools" to load this file. \quit
+
+CREATE OR REPLACE FUNCTION get_logs() RETURNS TABLE(
+    logtime timestamp with time zone,
+    loguser text,
+    logdatabase text,
+    logpid text,
+    logthread text,
+    loghost text,
+    logport text,
+    logsessiontime timestamp with time zone,
+    logtransaction int,
+    logsession text,
+    logcmdcount text,
+    logsegment text,
+    logslice text,
+    logdistxact text,
+    loglocalxact text,
+    logsubxact text,
+    logseverity text,
+    logstate text,
+    logmessage text,
+    logdetail text,
+    loghint text,
+    logquery text,
+    logquerypos int,
+    logcontext text,
+    logdebug text,
+    logcursorpos int,
+    logfunction text,
+    logfile text,
+    logline int,
+    logstack text
+) AS
+$BODY$
+DECLARE
+	r RECORD;
+BEGIN
+	CREATE TEMP TABLE my_logs(
+        logtime timestamp with time zone,
+        loguser text,
+        logdatabase text,
+        logpid text,
+        logthread text,
+        loghost text,
+        logport text,
+        logsessiontime timestamp with time zone,
+        logtransaction int,
+        logsession text,
+        logcmdcount text,
+        logsegment text,
+        logslice text,
+        logdistxact text,
+        loglocalxact text,
+        logsubxact text,
+        logseverity text,
+        logstate text,
+        logmessage text,
+        logdetail text,
+        loghint text,
+        logquery text,
+        logquerypos int,
+        logcontext text,
+        logdebug text,
+        logcursorpos int,
+        logfunction text,
+        logfile text,
+        logline int,
+        logstack text
+    ) DISTRIBUTED RANDOMLY;
+
+	FOR r IN
+		SELECT pg_ls_dir AS log_file FROM pg_ls_dir('pg_log') WHERE pg_ls_dir LIKE '%.csv' OR pg_ls_dir LIKE '%.log'
+	LOOP
+		EXECUTE $$COPY my_logs FROM 'pg_log/$$ || r.log_file || $$' CSV DELIMITER ',' QUOTE '"'$$;
+	END LOOP;
+	RETURN QUERY SELECT * FROM my_logs;
+    DROP TABLE my_logs;
+END
+$BODY$
+LANGUAGE plpgsql
+SECURITY DEFINER;
+
+CREATE VIEW gp_master_logs AS
+SELECT * FROM get_logs();

gpcontrib/gp_log_tools/gp_log_tools.control

@@ -0,0 +1,3 @@
+comment = ''
+default_version = '0.0.1'
+relocatable = true

gpcontrib/gp_log_tools/sql/gp_log_tools.sql

@@ -0,0 +1,19 @@
+CREATE EXTENSION gp_log_tools;
+
+SHOW ycmdb.yc_allow_copy_from_logs;
+
+SELECT count(*) != 0 as is_logs_selected FROM gp_master_logs;
+
+CREATE USER test_user1;
+
+GRANT SELECT ON gp_master_logs TO PUBLIC;
+
+SET SESSION AUTHORIZATION test_user1;
+
+SELECT count(*) != 0 as is_logs_selected FROM gp_master_logs;
+
+\c -
+
+DROP USER test_user1;
+
+DROP EXTENSION gp_log_tools;

src/backend/commands/copy.c

@@ -42,6 +42,7 @@
 #include "optimizer/clauses.h"
 #include "optimizer/planner.h"
 #include "parser/parse_relation.h"
+#include "postmaster/syslogger.h"
 #include "rewrite/rewriteHandler.h"
 #include "storage/fd.h"
 #include "tcop/tcopprot.h"
@@ -398,6 +399,8 @@ typedef struct
 bool yc_allow_copy_to_program;
 bool yc_allow_copy_to_file;
 bool yc_allow_copy_from_file;
+bool yc_allow_copy_from_logs;
+
 
 /*
  * Send copy start/stop messages for frontend copies.  These have changed
@@ -939,6 +942,45 @@ CopyLoadRawBuf(CopyState cstate)
 	return (inbytes > 0);
 }
 
+bool file_is_logs(const char *filepath)
+{	
+	char* filename;
+	char* abs_log_path;
+	bool is_logs = false;
+
+	if (filepath == NULL || DataDir == NULL)
+		return false;
+
+	filename = strdup(filepath);
+
+	canonicalize_path(filename);
+
+	if (path_contains_parent_reference(filename)) {
+		free(filename);
+		return false;
+	}
+
+	if (is_absolute_path(filename)) {
+		if (is_absolute_path(Log_directory)) {
+			abs_log_path = strdup(Log_directory);
+		} else {
+			abs_log_path = palloc(strlen(DataDir) + strlen(Log_directory) + 2);
+			join_path_components(abs_log_path, DataDir, Log_directory);
+		}
+
+		is_logs = path_is_prefix_of_path(abs_log_path, filename);
+		pfree(abs_log_path);
+	} else if (path_is_relative_and_below_cwd(filename)) {
+		is_logs = path_is_prefix_of_path(Log_directory, filename);
+	} else {
+		is_logs = false;
+	}
+
+	free(filename);
+
+	return is_logs;
+}
+
 
 /*
  *	 DoCopy executes the SQL COPY statement
@@ -971,6 +1013,7 @@ DoCopy(const CopyStmt *stmt, const char *queryString, uint64 *processed)
 	AclMode		required_access = (is_from ? ACL_INSERT : ACL_SELECT);
 	TupleDesc	tupDesc;
 	List	   *options;
+	bool		copy_from_logs;
 
 	glob_cstate = NULL;
 	glob_copystmt = (CopyStmt *) stmt;
@@ -997,8 +1040,10 @@ DoCopy(const CopyStmt *stmt, const char *queryString, uint64 *processed)
 			}
 		} else {
 			if (is_from) {
+				copy_from_logs = file_is_logs(stmt->filename);
+
 				// this is copy from file. This only could legitimately happen in initdb
-				if (!(superuser() && yc_allow_copy_from_file)) {
+				if (!(superuser() && (yc_allow_copy_from_file || (copy_from_logs && yc_allow_copy_from_logs)))) {
 					ereport(ERROR,
 								(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
 									errmsg("forbidden to COPY from file in Yandex Cloud"),

src/backend/utils/misc/guc.c

@@ -1644,6 +1644,15 @@ static struct config_bool ConfigureNamesBool[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"ycmdb.yc_allow_copy_from_logs", PGC_SUSET, DEVELOPER_OPTIONS,
+			gettext_noop("Whether to enable COPY from logs in Yandex Cloud"),
+		},
+		&yc_allow_copy_from_logs,
+		true,
+		NULL, NULL, NULL
+	},
+
 	/* End-of-list marker */
 	{
 		{NULL, 0, 0, NULL, NULL}, NULL, false, NULL, NULL, NULL

src/include/commands/copy.h

@@ -342,5 +342,6 @@ typedef struct GpDistributionData
 extern PGDLLIMPORT bool yc_allow_copy_to_program;
 extern PGDLLIMPORT bool yc_allow_copy_to_file;
 extern PGDLLIMPORT bool yc_allow_copy_from_file;
+extern PGDLLIMPORT bool yc_allow_copy_from_logs;
 
 #endif /* COPY_H */

src/include/utils/unsync_guc_name.h

@@ -563,3 +563,4 @@
 		"ycmdb.yc_allow_copy_to_program",
 		"ycmdb.yc_allow_copy_to_file",
 		"ycmdb.yc_allow_copy_from_file",
+		"ycmdb.yc_allow_copy_from_logs",

src/test/regress/GNUmakefile

@@ -165,6 +165,7 @@ submake-contrib-dummy_seclabel:
 .PHONY: hook-setup
 hook-setup:
 	gpconfig -c ycmdb.yc_allow_copy_from_file -v true
+	gpconfig -c ycmdb.yc_allow_copy_from_logs -v true
 	gpconfig -c ycmdb.yc_allow_copy_to_program -v true
 	gpconfig -c ycmdb.yc_allow_copy_to_file -v true