From 6ece6a3d93d4f55a17031630beb7759a26b76ad1 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 2 Jul 2021 15:28:52 -0400 Subject: [PATCH 001/126] fixed issue with malformed json and url in air_insert funciton --- R/airtabler.R | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/R/airtabler.R b/R/airtabler.R index c325c01..79525d4 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -287,10 +287,12 @@ air_insert <- function(base, table_name, record_data) { return( air_insert_data_frame(base, table_name, record_data)) } - record_data <- air_prepare_record(as.list(record_data)) - json_record_data <- jsonlite::toJSON(list(fields = record_data)) + fields <- list(fields = record_data) + records <- list(records = list(fields)) + json_record_data <- jsonlite::toJSON(records, pretty = F) request_url <- sprintf("%s/%s/%s", air_url, base, table_name) + request_url <- URLencode(request_url) # call service: res <- httr::POST( From 06da99724be934b2b6d779ab667280c12b97031c Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 6 Jul 2021 11:32:49 -0500 Subject: [PATCH 002/126] added preliminary functions for using the metadata schema request feautre in airtable. airtable is not currently taking requests for access to th metadata api so these functions have not been tested --- R/air_get_schema.R | 20 ++++++++++++++++++++ R/airtabler.R | 13 +++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 R/air_get_schema.R diff --git a/R/air_get_schema.R b/R/air_get_schema.R new file mode 100644 index 0000000..d5aa9e2 --- /dev/null +++ b/R/air_get_schema.R @@ -0,0 +1,20 @@ +air_get_schema <- function(base,...){ + request_url <- sprintf("%s/%s/tables", air_meta_url, base) + request_url <- URLencode(request_url) + + # call service: + res <- httr::GET( + request_url, + httr::add_headers( + Authorization = paste("Bearer", air_api_key()), + X-Airtable-Client-Secret = air_secret_key() + ) + ) + + air_validate(res) + # may need a new air_parse function + + schema <- jsonlite::fromJSON(res) + + return(schema) +} diff --git a/R/airtabler.R b/R/airtabler.R index 79525d4..1de4f37 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -12,7 +12,7 @@ #' #' \pkg{airtabler} functions will read the API key from #' environment variable \code{AIRTABLE_API_KEY}. To start R session with the -#' initialized environvent variable create an \code{.Renviron} file in your R home +#' initialized environment variable create an \code{.Renviron} file in your R home #' with a line like this: #' #' \code{AIRTABLE_API_KEY=************} @@ -29,6 +29,7 @@ NULL air_url <- "https://api.airtable.com/v0" +air_meta_url <- "https://api.airtable.com/v0/meta/bases" air_api_key <- function() { key <- Sys.getenv("AIRTABLE_API_KEY") @@ -38,8 +39,16 @@ air_api_key <- function() { key } +air_secret_key <- function(){ + key <- Sys.getenv("AIRTABLE_SECRET_KEY") + if(key == "") { + stop("AIRTABLE_SECRET_KEY environment variable is empty. See ?airtabler for help.") + } + key +} + -#' Get a list of records or retreive a single +#' Get a list of records or retrieve a single #' #' You can retrieve records in an order of a view by providing the name or ID of #' the view in the view query parameter. The results will include only records From f0d6e772b43383aa18a001d55345543e8b6f8364 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 6 Jul 2021 13:11:29 -0500 Subject: [PATCH 003/126] updated air_get to allow for subsetting by fields. This allows users to reduce the amount of data pulled in a query. --- R/airtabler.R | 22 +++++++++++++++++----- man/air_get.Rd | 25 +++++++++++++++++++------ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/R/airtabler.R b/R/airtabler.R index 1de4f37..c0c558c 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -66,20 +66,26 @@ air_secret_key <- function(){ #' @param sortField (optional) The field name to use for sorting #' @param sortDirection (optional) "asc" or "desc". The sort order in which the #' records will be returned. Defaults to asc. -#' @param combined_result If TRUE (default) all data is returned in the same data -#' frame. If FALSE table fields are returned in separate \code{fields} element. +#' @param combined_result If TRUE (default) all data is returned in the same data. +#' If FALSE table fields are returned in separate \code{fields} element. +#' @param fields (optional) Only data for fields whose names are in this list +#' will be included in the records. If you don't need every field, you can use +#' this parameter to reduce the amount of data transferred. #' @return A data frame with records or a list with record details if #' \code{record_id} is specified. #' @export -air_get <- function(base, table_name, record_id = NULL, +air_get <- function(base, table_name, + record_id = NULL, limit = NULL, offset = NULL, view = NULL, + fields = NULL, sortField = NULL, sortDirection = NULL, combined_result = TRUE) { search_path <- table_name + if(!missing(record_id)) { search_path <- paste0(search_path, "/", record_id) } @@ -90,7 +96,13 @@ air_get <- function(base, table_name, record_id = NULL, param_list <- as.list(environment())[c( "limit", "offset", "view", "sortField", "sortDirection")] param_list <- param_list[!sapply(param_list, is.null)] + if(!is.null(fields)) { + param_list <- c(param_list, list_params(x = fields, par_name = "fields")) + } + request_url <- httr::modify_url(request_url, query = param_list) + request_url <- gsub(pattern = "fields=",replacement = "fields%5B%5D=",x = request_url) + # call service: res <- httr::GET( url = request_url, @@ -159,7 +171,7 @@ list_params <- function(x, par_name) { #' @param table_name Table name #' @param record_id (optional) Use record ID argument to retrieve an existing #' record details -#' @param fields (optional) Only data for fields whose names are in this vector +#' @param fields (optional) Only data for fields whose names are in this list #' will be included in the records. If you don't need every field, you can use #' this parameter to reduce the amount of data transferred. #' @param filterByFormula (optional) A formula used to filter records. @@ -301,7 +313,7 @@ air_insert <- function(base, table_name, record_data) { json_record_data <- jsonlite::toJSON(records, pretty = F) request_url <- sprintf("%s/%s/%s", air_url, base, table_name) - request_url <- URLencode(request_url) + request_url <- utils::URLencode(request_url) # call service: res <- httr::POST( diff --git a/man/air_get.Rd b/man/air_get.Rd index 0f9d3f5..81b38cd 100644 --- a/man/air_get.Rd +++ b/man/air_get.Rd @@ -2,11 +2,20 @@ % Please edit documentation in R/airtabler.R \name{air_get} \alias{air_get} -\title{Get a list of records or retreive a single} +\title{Get a list of records or retrieve a single} \usage{ -air_get(base, table_name, record_id = NULL, limit = NULL, offset = NULL, - view = NULL, sortField = NULL, sortDirection = NULL, - combined_result = TRUE) +air_get( + base, + table_name, + record_id = NULL, + limit = NULL, + offset = NULL, + view = NULL, + fields = NULL, + sortField = NULL, + sortDirection = NULL, + combined_result = TRUE +) } \arguments{ \item{base}{Airtable base} @@ -24,13 +33,17 @@ call. Note that this is represented by a record ID, not a numerical offset.} \item{view}{(optional) The name or ID of the view} +\item{fields}{(optional) Only data for fields whose names are in this list +will be included in the records. If you don't need every field, you can use +this parameter to reduce the amount of data transferred.} + \item{sortField}{(optional) The field name to use for sorting} \item{sortDirection}{(optional) "asc" or "desc". The sort order in which the records will be returned. Defaults to asc.} -\item{combined_result}{If TRUE (default) all data is returned in the same data -frame. If FALSE table fields are returned in separate \code{fields} element.} +\item{combined_result}{If TRUE (default) all data is returned in the same data. +If FALSE table fields are returned in separate \code{fields} element.} } \value{ A data frame with records or a list with record details if From 79690f9ab75a260349e0bd14f74360d28b23c4e3 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 6 Jul 2021 13:12:06 -0500 Subject: [PATCH 004/126] updated documentation to specify that fields must be given as a list --- man/air_select.Rd | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/man/air_select.Rd b/man/air_select.Rd index 00fa4ef..5b246c2 100644 --- a/man/air_select.Rd +++ b/man/air_select.Rd @@ -4,9 +4,19 @@ \alias{air_select} \title{Select} \usage{ -air_select(base, table_name, record_id = NULL, fields = NULL, - filterByFormula = NULL, maxRecord = NULL, sort = NULL, view = NULL, - pageSize = NULL, offset = NULL, combined_result = TRUE) +air_select( + base, + table_name, + record_id = NULL, + fields = NULL, + filterByFormula = NULL, + maxRecord = NULL, + sort = NULL, + view = NULL, + pageSize = NULL, + offset = NULL, + combined_result = TRUE +) } \arguments{ \item{base}{Airtable base} @@ -16,7 +26,7 @@ air_select(base, table_name, record_id = NULL, fields = NULL, \item{record_id}{(optional) Use record ID argument to retrieve an existing record details} -\item{fields}{(optional) Only data for fields whose names are in this vector +\item{fields}{(optional) Only data for fields whose names are in this list will be included in the records. If you don't need every field, you can use this parameter to reduce the amount of data transferred.} From 05cfc6ad3743cf0817976722f4163c5fc9f04da2 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 6 Jul 2021 13:13:07 -0500 Subject: [PATCH 005/126] added a provisional function for querying the meta data api --- R/air_get_schema.R | 18 ++++++++++++++++-- man/air_get_schema.Rd | 25 +++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 man/air_get_schema.Rd diff --git a/R/air_get_schema.R b/R/air_get_schema.R index d5aa9e2..46e19be 100644 --- a/R/air_get_schema.R +++ b/R/air_get_schema.R @@ -1,13 +1,27 @@ +#' Get base schema +#' +#' Get the schema for the tables in a base. +#' +#' @section Using Metadata API: +#' The meta data api is not accepting new requests for client secrets as of +#' 06 July 2021. Will update when metadata api becomes available. +#' +#' @param base Airtable base ID +#' @param ... additional paramters +#' +#' @return list of schema +#' @export air_get_schema + air_get_schema <- function(base,...){ request_url <- sprintf("%s/%s/tables", air_meta_url, base) - request_url <- URLencode(request_url) + request_url <- utils::URLencode(request_url) # call service: res <- httr::GET( request_url, httr::add_headers( Authorization = paste("Bearer", air_api_key()), - X-Airtable-Client-Secret = air_secret_key() + "X-Airtable-Client-Secret" = air_secret_key() ) ) diff --git a/man/air_get_schema.Rd b/man/air_get_schema.Rd new file mode 100644 index 0000000..61782d5 --- /dev/null +++ b/man/air_get_schema.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_get_schema.R +\name{air_get_schema} +\alias{air_get_schema} +\title{Get base schema} +\usage{ +air_get_schema(base, ...) +} +\arguments{ +\item{base}{Airtable base ID} + +\item{...}{additional paramters} +} +\value{ +list of schema +} +\description{ +Get the schema for the tables in a base. +} +\section{Using Metadata API}{ + +The meta data api is not accepting new requests for client secrets as of +06 July 2021. Will update when metadata api becomes available. +} + From 8ffbc8259ba8240f75f1c6acbe158fe653c9c12d Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 6 Jul 2021 13:13:53 -0500 Subject: [PATCH 006/126] updated package documentation --- DESCRIPTION | 2 +- NAMESPACE | 1 + man/airtabler-package.Rd | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index fe2dd0a..3d983a5 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -14,4 +14,4 @@ LazyData: TRUE Imports: httr, jsonlite -RoxygenNote: 6.0.1 +RoxygenNote: 7.1.1 diff --git a/NAMESPACE b/NAMESPACE index 96761eb..26e5f45 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,6 +3,7 @@ S3method(print,airtable.base) export(air_delete) export(air_get) +export(air_get_schema) export(air_insert) export(air_select) export(air_update) diff --git a/man/airtabler-package.Rd b/man/airtabler-package.Rd index aae9d1e..0e0cb64 100644 --- a/man/airtabler-package.Rd +++ b/man/airtabler-package.Rd @@ -21,7 +21,7 @@ Provides access to the Airtable API (\url{http://airtable.com/api}). \pkg{airtabler} functions will read the API key from environment variable \code{AIRTABLE_API_KEY}. To start R session with the - initialized environvent variable create an \code{.Renviron} file in your R home + initialized environment variable create an \code{.Renviron} file in your R home with a line like this: \code{AIRTABLE_API_KEY=************} From 570bca3fe4b1cac51d7dd00d5cf2a7b5bcbe119b Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 29 Jul 2021 12:40:51 -0500 Subject: [PATCH 007/126] added fetch_all to airtabl. Changed params to make them consistent with airtableR functions --- DESCRIPTION | 2 ++ NAMESPACE | 1 + R/fetch_all.R | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ man/fetch_all.Rd | 52 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 R/fetch_all.R create mode 100644 man/fetch_all.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 3d983a5..2eac63c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -12,6 +12,8 @@ License: MIT + file LICENSE URL: https://github.com/bergant/airtabler LazyData: TRUE Imports: + glue, + dplyr, httr, jsonlite RoxygenNote: 7.1.1 diff --git a/NAMESPACE b/NAMESPACE index 26e5f45..41edafa 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -8,5 +8,6 @@ export(air_insert) export(air_select) export(air_update) export(airtable) +export(fetch_all) export(get_offset) export(multiple) diff --git a/R/fetch_all.R b/R/fetch_all.R new file mode 100644 index 0000000..87cccb8 --- /dev/null +++ b/R/fetch_all.R @@ -0,0 +1,57 @@ +#' Fetch All Records in an Airtable +#' +#' Airtable limits the number of records that can be pulled from a base to 100. +#' This function pulls records based on a query then checks if there is an +#' offset value. While there is an offset value, it uses that value to generate +#' the next query, thus moving down the records until all records have been +#' fetched from the database. +#' +#' @section Adding airtable API key to your environment: +#' Airtable requires an api key to fetch data. +#' Generate the airtable API key from your [Airtable account](http://airtable.com/account) page. +#' +#' \code{airtabler} functions will read the API key from +#' environment variable \code{AIRTABLE_API_KEY}. To start R session with the +#' initialized environment variable create an \code{.Renviron} file in your +#' home directory with a line like this: +#' +#' \code{AIRTABLE_API_KEY=your_api_key_here} +#' +#' You can use \code{usethis::edit_r_environ()} to edit your find and edit your +#' file. +#' +#' @param base String. ID for the base or app to be fetched +#' @param table_name String. Name of the table to be fetched from the base +#' @param ... Additional arguments to pass to \code{air_get}. \code{view} is a +#' commonly used additional argument. +#' +#' @return dataframe +#' @export fetch_all +#' +#' @examples +#' # Each base a fully described API +#' app_id <- "appwlxIzmQx5njRtQ" # ID for the base we are fetching. +#' # Note that you can pass a `view` argument to air_get or fetch_all to get only +#' # a view of a table (say, only validated records, or some other filtered view), +#' # e.g., +#' # bats <- fetch_all(app_id, "Bat", view = "Validated Records") +#' talks <- fetch_all(app_id, "TALKS") +#' +fetch_all <- function(base, table_name, ...) { + out <- list() + out[[1]] <- airtabler::air_get(base, table_name, combined_result = FALSE,...) + if(length(out[[1]]) == 0){ + emptyTableMessage <- glue::glue("The queried view for {table_name} in {base} is empty") + warning(emptyTableMessage) + return(emptyTableMessage) + } else { + offset <- airtabler::get_offset(out[[1]]) + while (!is.null(offset)) { + out <- c(out, list(airtabler::air_get(base, table_name, combined_result = FALSE, offset = offset, ...))) + offset <- airtabler::get_offset(out[[length(out)]]) + } + out <- dplyr::bind_rows(out) + cbind(id = out$id, out$fields, createdTime = out$createdTime, + stringsAsFactors = FALSE) + } +} diff --git a/man/fetch_all.Rd b/man/fetch_all.Rd new file mode 100644 index 0000000..320b895 --- /dev/null +++ b/man/fetch_all.Rd @@ -0,0 +1,52 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/fetch_all.R +\name{fetch_all} +\alias{fetch_all} +\title{Fetch All Records in an Airtable} +\usage{ +fetch_all(base, table_name, ...) +} +\arguments{ +\item{base}{String. ID for the base or app to be fetched} + +\item{table_name}{String. Name of the table to be fetched from the base} + +\item{...}{Additional arguments to pass to \code{air_get}. \code{view} is a +commonly used additional argument.} +} +\value{ +dataframe +} +\description{ +Airtable limits the number of records that can be pulled from a base to 100. +This function pulls records based on a query then checks if there is an +offset value. While there is an offset value, it uses that value to generate +the next query, thus moving down the records until all records have been +fetched from the database. +} +\section{Adding airtable API key to your environment}{ + +Airtable requires an api key to fetch data. +Generate the airtable API key from your [Airtable account](http://airtable.com/account) page. + +\code{airtabler} functions will read the API key from +environment variable \code{AIRTABLE_API_KEY}. To start R session with the +initialized environment variable create an \code{.Renviron} file in your +home directory with a line like this: + +\code{AIRTABLE_API_KEY=your_api_key_here} + +You can use \code{usethis::edit_r_environ()} to edit your find and edit your +file. +} + +\examples{ +# Each base a fully described API +app_id <- "appwlxIzmQx5njRtQ" # ID for the base we are fetching. +# Note that you can pass a `view` argument to air_get or fetch_all to get only +# a view of a table (say, only validated records, or some other filtered view), +# e.g., +# bats <- fetch_all(app_id, "Bat", view = "Validated Records") +talks <- fetch_all(app_id, "TALKS") + +} From 0c5c1e6ff96287f67019ed0fccb0eb3c66995e97 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 2 Aug 2021 12:32:05 -0500 Subject: [PATCH 008/126] moved functions from nipah bangladesh to airtabler pacakge --- DESCRIPTION | 5 +++- NAMESPACE | 2 ++ R/air_get_attachments.R | 49 ++++++++++++++++++++++++++++++++++++++ R/read_excel_url.R | 16 +++++++++++++ man/air_get_attachments.Rd | 35 +++++++++++++++++++++++++++ man/read_excel_url.Rd | 21 ++++++++++++++++ 6 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 R/air_get_attachments.R create mode 100644 R/read_excel_url.R create mode 100644 man/air_get_attachments.Rd create mode 100644 man/read_excel_url.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 2eac63c..5f14e43 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -15,5 +15,8 @@ Imports: glue, dplyr, httr, - jsonlite + jsonlite, + readxl, + curl, + purrr RoxygenNote: 7.1.1 diff --git a/NAMESPACE b/NAMESPACE index 41edafa..6d5a55d 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,6 +3,7 @@ S3method(print,airtable.base) export(air_delete) export(air_get) +export(air_get_attachments) export(air_get_schema) export(air_insert) export(air_select) @@ -11,3 +12,4 @@ export(airtable) export(fetch_all) export(get_offset) export(multiple) +export(read_excel_url) diff --git a/R/air_get_attachments.R b/R/air_get_attachments.R new file mode 100644 index 0000000..8c886f8 --- /dev/null +++ b/R/air_get_attachments.R @@ -0,0 +1,49 @@ +#' Get Airtable file attachments +#' +#' Get an attachment stored in air tables. For excel files, returns a named list. +#' +#' @param base String. ID for the base or app to be fetched +#' @param table_name String. Name of the table to be fetched from the base +#' @param field String. Name of field with file attachments in base +#' @param extract_type String. File type to be extracted. +#' Should be one of: excel +#' @param extract_field String. Name of extract field that will be created +#' @param ... Additional arguments to pass to \code{air_get} +#' +#' +#' @return named list of data frames +#' @export air_get_attachments +#' +#' @examples +air_get_attachments <- function(base, table_name, field, extract_type ="excel", extract_field ="excel_extract", ...){ + #browser() + # get data + x <- fetch_all(base,table_name,...) + + ### subset to necessary records ---- + + # get files + xfield <- purrr::pluck(x,field) + + ### get files ---- + if(extract_type == "excel"){ + + xlist <- purrr::map(xfield,function(x){ + if(is.null(x$url)){ + ID <- x$id + warning(sprintf("Record ID %s is null",ID)) + return(NULL) + } + read_excel_url(x$url) ## need to be able to pass additional arguments + }) + + ## add extract to data frame ---- + x[[extract_field]] <- xlist + } + + return(x) +} + + + + diff --git a/R/read_excel_url.R b/R/read_excel_url.R new file mode 100644 index 0000000..c0ea6dc --- /dev/null +++ b/R/read_excel_url.R @@ -0,0 +1,16 @@ +#' Read an excel file from URL +#' +#' Extends \code{readxl::read_excel} to allow for reading from a URL. +#' @param url String. Url for file +#' @param fileext String. File extension for temp file +#' @param ... additional arguments to pass to \code{read_excel} +#' +#' @return tibble +#' @export read_excel_url +#' +#' @examples +read_excel_url <- function(url, fileext= ".xslx",...){ + tmp <- tempfile(fileext = ".xslx") + curl::curl_download(url, tmp ) + readxl::read_excel(tmp,...) +} diff --git a/man/air_get_attachments.Rd b/man/air_get_attachments.Rd new file mode 100644 index 0000000..7314c60 --- /dev/null +++ b/man/air_get_attachments.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_get_attachments.R +\name{air_get_attachments} +\alias{air_get_attachments} +\title{Get Airtable file attachments} +\usage{ +air_get_attachments( + base, + table_name, + field, + extract_type = "excel", + extract_field = "excel_extract", + ... +) +} +\arguments{ +\item{base}{String. ID for the base or app to be fetched} + +\item{table_name}{String. Name of the table to be fetched from the base} + +\item{field}{String. Name of field with file attachments in base} + +\item{extract_type}{String. File type to be extracted. +Should be one of: excel} + +\item{extract_field}{String. Name of extract field that will be created} + +\item{...}{Additional arguments to pass to \code{air_get}} +} +\value{ +named list of data frames +} +\description{ +Get an attachment stored in air tables. For excel files, returns a named list. +} diff --git a/man/read_excel_url.Rd b/man/read_excel_url.Rd new file mode 100644 index 0000000..d41a521 --- /dev/null +++ b/man/read_excel_url.Rd @@ -0,0 +1,21 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/read_excel_url.R +\name{read_excel_url} +\alias{read_excel_url} +\title{Read an excel file from URL} +\usage{ +read_excel_url(url, fileext = ".xslx", ...) +} +\arguments{ +\item{url}{String. Url for file} + +\item{fileext}{String. File extension for temp file} + +\item{...}{additional arguments to pass to \code{read_excel}} +} +\value{ +tibble +} +\description{ +Extends \code{readxl::read_excel} to allow for reading from a URL. +} From 12d7322f2627a7cb27b8325bf40e822e93d18e5e Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 2 Aug 2021 14:37:44 -0500 Subject: [PATCH 009/126] added a function to get unique field values --- R/get_unique_field_values.R | 37 ++++++++++++++++++++++++++++++++++ man/get_unique_field_values.Rd | 26 ++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 R/get_unique_field_values.R create mode 100644 man/get_unique_field_values.Rd diff --git a/R/get_unique_field_values.R b/R/get_unique_field_values.R new file mode 100644 index 0000000..ea3b990 --- /dev/null +++ b/R/get_unique_field_values.R @@ -0,0 +1,37 @@ +#' Get unique values from a field +#' +#' Because the airtable api lacks an easy solution for getting +#' unique values from a field in a table, we have this function +#' which accepts a list of fields and returns their unique +#' values. Currently, if multiple fields are listed, all unique +#' values from all fields will be returned in a single vector. +#' This may change in future iterations. +#' +#' +#' @param base String. ID of airtable base +#' @param table_name String. Name of table in base +#' @param fields List. Names of fields +#' +#' @return vector of unique values +#' @export +#' +#' @examples +get_unique_field_values <- function(base, + table_name, + fields){ + + baseField <- airtabler::fetch_all(base, table_name , fields = fields ) + + if(is.character(baseField)){ + return("") + } else { + + fieldsVector <- unlist(fields) + + uniqueField <- unlist(unique(baseField[fieldsVector])) + + return(uniqueField) + + } + +} diff --git a/man/get_unique_field_values.Rd b/man/get_unique_field_values.Rd new file mode 100644 index 0000000..8758aa7 --- /dev/null +++ b/man/get_unique_field_values.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_unique_field_values.R +\name{get_unique_field_values} +\alias{get_unique_field_values} +\title{Get unique values from a field} +\usage{ +get_unique_field_values(base, table_name, fields) +} +\arguments{ +\item{base}{String. ID of airtable base} + +\item{table_name}{String. Name of table in base} + +\item{fields}{List. Names of fields} +} +\value{ +vector of unique values +} +\description{ +Because the airtable api lacks an easy solution for getting +unique values from a field in a table, we have this function +which accepts a list of fields and returns their unique +values. Currently, if multiple fields are listed, all unique +values from all fields will be returned in a single vector. +This may change in future iterations. +} From 240d5b79b201695bae150fae86de46c03355886d Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 2 Aug 2021 14:38:40 -0500 Subject: [PATCH 010/126] explicitly named base and table_name in make json function --- R/airtabler.R | 53 +++++++++++++++++++++++++++++++++-------------- man/air_insert.Rd | 9 ++++++-- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/R/airtabler.R b/R/airtabler.R index c0c558c..20a3894 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -302,38 +302,61 @@ air_parse <- function(res) { #' @param record_data Named list of values. You can include all, some, or none #' of the field values #' @export +#' +#' air_insert <- function(base, table_name, record_data) { if( inherits(record_data, "data.frame")) { return( air_insert_data_frame(base, table_name, record_data)) } + # create json records + json_record_data <- air_make_json(base, table_name, record_data) + + # call service: + air_make_request(base,table_name,json_record_data) +} + + +air_make_json <- function (base, table_name, record_data){ + if (inherits(record_data, "data.frame")) { + return(air_insert_data_frame(base, table_name, record_data)) + } + record_data <- air_prepare_record(as.list(record_data)) fields <- list(fields = record_data) records <- list(records = list(fields)) - json_record_data <- jsonlite::toJSON(records, pretty = F) + json_record_data <- jsonlite::toJSON(records, pretty = T) + return(json_record_data) +} + + +air_make_request <- function(base, table_name, json_record_data){ request_url <- sprintf("%s/%s/%s", air_url, base, table_name) request_url <- utils::URLencode(request_url) - # call service: - res <- httr::POST( - request_url, - httr::add_headers( - Authorization = paste("Bearer", air_api_key()), - `Content-type` = "application/json" - ), - body = json_record_data - ) + res <- httr::POST(url = request_url, + httr::add_headers( + Authorization = paste("Bearer",air_api_key()), + 'Content-type' = "application/json"), + body = json_record_data) - air_validate(res) # throws exception (stop) if error - air_parse(res) # returns R object + air_validate(res) # throws exception (stop) if error + air_parse(res) # returns R object } +#' @param base String. Airtable base +#' @param table_name String. Table name +#' @param records Dataframe. Contains records you would like to insert +#' +#' @rdname air_insert +#' @export air_insert_data_frame air_insert_data_frame <- function(base, table_name, records) { lapply(seq_len(nrow(records)), function(i) { - record_data <- - as.list(records[i,]) - air_insert(base = base, table_name = table_name, record_data = record_data) + record_data <- as.list(records[i,]) + json_record_data <- air_make_json(base, table_name, record_data) + air_make_request(base = base,table_name = table_name ,json_record_data = json_record_data ) + }) } diff --git a/man/air_insert.Rd b/man/air_insert.Rd index 6e22eba..20cfd59 100644 --- a/man/air_insert.Rd +++ b/man/air_insert.Rd @@ -2,21 +2,26 @@ % Please edit documentation in R/airtabler.R \name{air_insert} \alias{air_insert} +\alias{air_insert_data_frame} \alias{multiple} \title{Insert a new record} \usage{ air_insert(base, table_name, record_data) +air_insert_data_frame(base, table_name, records) + multiple(x) } \arguments{ -\item{base}{Airtable base} +\item{base}{String. Airtable base} -\item{table_name}{Table name} +\item{table_name}{String. Table name} \item{record_data}{Named list of values. You can include all, some, or none of the field values} +\item{records}{Dataframe. Contains records you would like to insert} + \item{x}{Object to be marked as a multiple value field} } \description{ From 7eea06a3e47afc6c7f6e4baab91d42f8c6476769 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 2 Aug 2021 14:39:05 -0500 Subject: [PATCH 011/126] updating namespace --- NAMESPACE | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NAMESPACE b/NAMESPACE index 6d5a55d..8e49701 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,10 +6,12 @@ export(air_get) export(air_get_attachments) export(air_get_schema) export(air_insert) +export(air_insert_data_frame) export(air_select) export(air_update) export(airtable) export(fetch_all) export(get_offset) +export(get_unique_field_values) export(multiple) export(read_excel_url) From 3b32eba270b467ea79dbeb299af42f1df11581b0 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 28 Sep 2021 11:36:44 -0400 Subject: [PATCH 012/126] updated air_update funciton to use make json and make request functions --- R/airtabler.R | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/R/airtabler.R b/R/airtabler.R index 20a3894..b151fe4 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -314,7 +314,7 @@ air_insert <- function(base, table_name, record_data) { json_record_data <- air_make_json(base, table_name, record_data) # call service: - air_make_request(base,table_name,json_record_data) + air_make_request(base,table_name,json_record_data, method = "POST") } @@ -330,16 +330,28 @@ air_make_json <- function (base, table_name, record_data){ } -air_make_request <- function(base, table_name, json_record_data){ +air_make_request <- function(base, table_name, json_record_data, method = c("POST","PATCH")){ request_url <- sprintf("%s/%s/%s", air_url, base, table_name) request_url <- utils::URLencode(request_url) + if(method == "POST"){ + res <- httr::POST(url = request_url, httr::add_headers( Authorization = paste("Bearer",air_api_key()), 'Content-type' = "application/json"), body = json_record_data) + } + + if(method == "PATCH"){ + + res <- httr::PATCH(url = request_url, + httr::add_headers( + Authorization = paste("Bearer",air_api_key()), + 'Content-type' = "application/json"), + body = json_record_data) + } air_validate(res) # throws exception (stop) if error air_parse(res) # returns R object @@ -447,23 +459,12 @@ air_update <- function(base, table_name, record_id, record_data) { if(inherits(record_data, "data.frame")) { return(air_update_data_frame(base, table_name, record_id, record_data)) } - record_data <- air_prepare_record(record_data) - json_record_data <- jsonlite::toJSON(list(fields = record_data)) - request_url <- sprintf("%s/%s/%s/%s", air_url, base, table_name, record_id) + #create json records + json_record_data <- air_make_json(base, table_name, record_data) # call service: - res <- httr::PATCH( - request_url, - httr::add_headers( - Authorization = paste("Bearer", air_api_key()), - `Content-type` = "application/json" - ), - body = json_record_data - ) - - air_validate(res) # throws exception (stop) if error - air_parse(res) # returns R object + air_make_request(base,table_name,json_record_data, method = "PATCH") } #' Get airtable base object From f367a5229f3781628936b0436dab40e51eb88242 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 28 Sep 2021 11:45:09 -0400 Subject: [PATCH 013/126] added record id to request url for PATCH method --- R/airtabler.R | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/R/airtabler.R b/R/airtabler.R index b151fe4..d4dd126 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -332,20 +332,25 @@ air_make_json <- function (base, table_name, record_data){ air_make_request <- function(base, table_name, json_record_data, method = c("POST","PATCH")){ - request_url <- sprintf("%s/%s/%s", air_url, base, table_name) - request_url <- utils::URLencode(request_url) + if(method == "POST"){ - res <- httr::POST(url = request_url, - httr::add_headers( - Authorization = paste("Bearer",air_api_key()), - 'Content-type' = "application/json"), - body = json_record_data) + request_url <- sprintf("%s/%s/%s", air_url, base, table_name) + request_url <- utils::URLencode(request_url) + + res <- httr::POST(url = request_url, + httr::add_headers( + Authorization = paste("Bearer",air_api_key()), + 'Content-type' = "application/json"), + body = json_record_data) } if(method == "PATCH"){ + request_url <- sprintf("%s/%s/%s/%s", air_url, base, table_name, record_id) + request_url <- utils::URLencode(request_url) + res <- httr::PATCH(url = request_url, httr::add_headers( Authorization = paste("Bearer",air_api_key()), From e6a3358b0a925ecaeae77980466b14921dc121ff Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 28 Sep 2021 14:20:13 -0400 Subject: [PATCH 014/126] added record id and methods to air_make request, added methods to update and insert functions, and refactored updated functions to use make json and make request functions --- R/airtabler.R | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/R/airtabler.R b/R/airtabler.R index d4dd126..151588f 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -330,9 +330,7 @@ air_make_json <- function (base, table_name, record_data){ } -air_make_request <- function(base, table_name, json_record_data, method = c("POST","PATCH")){ - - +air_make_request <- function(base, table_name, json_record_data, record_id = NULL, method = c("POST","PATCH")){ if(method == "POST"){ @@ -372,19 +370,19 @@ air_insert_data_frame <- function(base, table_name, records) { lapply(seq_len(nrow(records)), function(i) { record_data <- as.list(records[i,]) json_record_data <- air_make_json(base, table_name, record_data) - air_make_request(base = base,table_name = table_name ,json_record_data = json_record_data ) + air_make_request(base = base,table_name = table_name ,json_record_data = json_record_data, method = "POST" ) }) } air_update_data_frame <- function(base, table_name, record_ids, records) { lapply(seq_len(nrow(records)), function(i) { - record_data <- - unlist(as.list(records[i,]), recursive = FALSE) - air_update(base = base, - table_name = table_name, - record_id = ifelse(is.null(record_ids), record_data$id, record_ids[i]), - record_data = record_data) + record_data <- as.list(records[i,]) + json_record_data <- air_make_json(base, table_name, record_data) + air_make_request(base = base,table_name = table_name, + json_record_data = json_record_data, + record_id = ifelse(is.null(record_ids), record_data$id, record_ids[i]), + method = "PATCH") }) } @@ -469,7 +467,7 @@ air_update <- function(base, table_name, record_id, record_data) { json_record_data <- air_make_json(base, table_name, record_data) # call service: - air_make_request(base,table_name,json_record_data, method = "PATCH") + air_make_request(base,table_name,json_record_data,record_id = record_id, method = "PATCH") } #' Get airtable base object From 43c72a52c507c1e92af084cc3cc052d9604a391a Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 28 Sep 2021 20:48:50 -0400 Subject: [PATCH 015/126] added methods for PATCH calls to make_json and make_request, added typecast option for make_json, got update_dataframe and update functions to work. --- R/airtabler.R | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/R/airtabler.R b/R/airtabler.R index 151588f..b452bb8 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -318,12 +318,33 @@ air_insert <- function(base, table_name, record_data) { } -air_make_json <- function (base, table_name, record_data){ +air_make_json <- function (base, table_name, record_data, record_id = NULL, method = NULL,typecast = TRUE){ if (inherits(record_data, "data.frame")) { return(air_insert_data_frame(base, table_name, record_data)) } + + #browser() record_data <- air_prepare_record(as.list(record_data)) fields <- list(fields = record_data) + + if(typecast){ + # allows us to use values not currently specified in the + # table or record eg issues = "some new issue" + # use unbox to create singleton values instead of json objects + fields$typecast <- jsonlite::unbox(typecast) + } + + if(method == "PATCH"){ + ## the patch method already specifies a record so we + ## can remove a lot of addition json elements + + ## drop the id field as it can't be updated + fields$fields$id <- NULL + + json_patch_data <- jsonlite::toJSON(fields, pretty = T) + return(json_patch_data) + } + records <- list(records = list(fields)) json_record_data <- jsonlite::toJSON(records, pretty = T) return(json_record_data) @@ -346,9 +367,16 @@ air_make_request <- function(base, table_name, json_record_data, record_id = NUL if(method == "PATCH"){ + ### Because the patch request url specifies the record, + ### the json does not need to be as complete + request_url <- sprintf("%s/%s/%s/%s", air_url, base, table_name, record_id) request_url <- utils::URLencode(request_url) + print(request_url) + + browser() + res <- httr::PATCH(url = request_url, httr::add_headers( Authorization = paste("Bearer",air_api_key()), @@ -378,10 +406,13 @@ air_insert_data_frame <- function(base, table_name, records) { air_update_data_frame <- function(base, table_name, record_ids, records) { lapply(seq_len(nrow(records)), function(i) { record_data <- as.list(records[i,]) - json_record_data <- air_make_json(base, table_name, record_data) + record_id <- ifelse(is.null(record_ids), record_data$id, record_ids[i]) + json_record_data <- air_make_json(base, table_name, record_data, + record_id = record_id,method = "PATCH") + air_make_request(base = base,table_name = table_name, json_record_data = json_record_data, - record_id = ifelse(is.null(record_ids), record_data$id, record_ids[i]), + record_id = record_id, method = "PATCH") }) } @@ -459,15 +490,19 @@ air_delete_vec <- Vectorize(air_delete, vectorize.args = "record_id", SIMPLIFY = #' @export air_update <- function(base, table_name, record_id, record_data) { + method <- "PATCH" + if(inherits(record_data, "data.frame")) { + print("using data.frame method") return(air_update_data_frame(base, table_name, record_id, record_data)) } #create json records - json_record_data <- air_make_json(base, table_name, record_data) + json_record_data <- air_make_json(base, table_name, record_data, + record_id = record_id, method = method) # call service: - air_make_request(base,table_name,json_record_data,record_id = record_id, method = "PATCH") + air_make_request(base,table_name,json_record_data,record_id = record_id, method = method) } #' Get airtable base object From 6416456405cb741e259fb234c5db6d7bf260bf16 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 28 Sep 2021 20:57:05 -0400 Subject: [PATCH 016/126] removed browser and print statements --- R/airtabler.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R/airtabler.R b/R/airtabler.R index b452bb8..080126e 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -373,9 +373,9 @@ air_make_request <- function(base, table_name, json_record_data, record_id = NUL request_url <- sprintf("%s/%s/%s/%s", air_url, base, table_name, record_id) request_url <- utils::URLencode(request_url) - print(request_url) + #print(request_url) - browser() + # browser() res <- httr::PATCH(url = request_url, httr::add_headers( @@ -493,7 +493,7 @@ air_update <- function(base, table_name, record_id, record_data) { method <- "PATCH" if(inherits(record_data, "data.frame")) { - print("using data.frame method") + #print("using data.frame method") return(air_update_data_frame(base, table_name, record_id, record_data)) } From c3548ad3f734c5cbe2da6e04b81e36276599f6c2 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 5 Oct 2021 19:52:53 -0400 Subject: [PATCH 017/126] added a delete method to the make request function and updated air_delete --- R/airtabler.R | 51 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/R/airtabler.R b/R/airtabler.R index 080126e..c77bad6 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -351,7 +351,21 @@ air_make_json <- function (base, table_name, record_data, record_id = NULL, meth } -air_make_request <- function(base, table_name, json_record_data, record_id = NULL, method = c("POST","PATCH")){ +#' Make an HTTP request +#' +#' Properly encodes HTTP requests +#' +#' @param base +#' @param table_name +#' @param json_record_data +#' @param record_id +#' @param method +#' +#' @return +#' @export +#' +#' @examples +air_make_request <- function(base, table_name, json_record_data, record_id = NULL, method = c("POST","PATCH","DELETE")){ if(method == "POST"){ @@ -384,6 +398,26 @@ air_make_request <- function(base, table_name, json_record_data, record_id = NUL body = json_record_data) } + if(method == "DELETE"){ + + ### Because the patch request url specifies the record, + ### the json does not need to be as complete + + request_url <- sprintf("%s/%s/%s/%s", air_url, base, table_name, record_id) + request_url <- utils::URLencode(request_url) + + # call service: + res <- httr::DELETE( + request_url, + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ) + ) + + air_validate(res) # throws exception (stop) if error + air_parse(res) # returns R object + } + air_validate(res) # throws exception (stop) if error air_parse(res) # returns R object } @@ -458,18 +492,11 @@ air_delete <- function(base, table_name, record_id) { return(air_delete_vec(base, table_name, record_id)) } - request_url <- sprintf("%s/%s/%s/%s", air_url, base, table_name, record_id) - - # call service: - res <- httr::DELETE( - request_url, - httr::add_headers( - Authorization = paste("Bearer", air_api_key()) - ) - ) + air_make_request(base = base, + table_name = table_name, + record_id = record_id, + method = "DELETE") - air_validate(res) # throws exception (stop) if error - air_parse(res) # returns R object } air_delete_vec <- Vectorize(air_delete, vectorize.args = "record_id", SIMPLIFY = FALSE) From a3efd0ca458c01f2b55359a16e0b5d958b796c04 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 5 Oct 2021 20:07:30 -0400 Subject: [PATCH 018/126] documented make_request and maKe_json functions --- R/airtabler.R | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/R/airtabler.R b/R/airtabler.R index c77bad6..c543123 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -318,6 +318,22 @@ air_insert <- function(base, table_name, record_data) { } +#' Make JSON for API +#' +#' Make JSON that is compatible with the Airtable API. +#' +#' @param base String. Base in airtable +#' @param table_name String. Table in airtable +#' @param record_data Dataframe, list, or vector. Data to be converted to JSON +#' @param record_id String or vector of strings. Records to be manipulated +#' @param method String. "PATCH" is necessary for \code{air_update} +#' @param typecast Logical. Should the typecast option be TRUE or FALSE? Typecast +#' allows you to add new options to select type fields. +#' +#' @return +#' @export +#' +#' @examples air_make_json <- function (base, table_name, record_data, record_id = NULL, method = NULL,typecast = TRUE){ if (inherits(record_data, "data.frame")) { return(air_insert_data_frame(base, table_name, record_data)) @@ -355,11 +371,11 @@ air_make_json <- function (base, table_name, record_data, record_id = NULL, meth #' #' Properly encodes HTTP requests #' -#' @param base -#' @param table_name -#' @param json_record_data -#' @param record_id -#' @param method +#' @param base String. Base in airtable +#' @param table_name String. Table in airtable +#' @param json_record_data json or string. JSON formatted text with record data +#' @param record_id String or vector of strings. Record id +#' @param method String. One of "POST", "PATCH", or "DELETE" #' #' @return #' @export From 406522bf9d6886e4eba24a72e25281df8291a944 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 5 Oct 2021 21:01:03 -0400 Subject: [PATCH 019/126] updated documentation and added a post method to make json --- NAMESPACE | 2 ++ R/airtabler.R | 9 +++------ man/air_make_json.Rd | 35 +++++++++++++++++++++++++++++++++++ man/air_make_request.Rd | 31 +++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 man/air_make_json.Rd create mode 100644 man/air_make_request.Rd diff --git a/NAMESPACE b/NAMESPACE index 8e49701..2544008 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -7,6 +7,8 @@ export(air_get_attachments) export(air_get_schema) export(air_insert) export(air_insert_data_frame) +export(air_make_json) +export(air_make_request) export(air_select) export(air_update) export(airtable) diff --git a/R/airtabler.R b/R/airtabler.R index c543123..2b876c3 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -330,11 +330,11 @@ air_insert <- function(base, table_name, record_data) { #' @param typecast Logical. Should the typecast option be TRUE or FALSE? Typecast #' allows you to add new options to select type fields. #' -#' @return +#' @return JSON with record data #' @export #' #' @examples -air_make_json <- function (base, table_name, record_data, record_id = NULL, method = NULL,typecast = TRUE){ +air_make_json <- function (base, table_name, record_data, record_id = NULL, method = "POST",typecast = TRUE){ if (inherits(record_data, "data.frame")) { return(air_insert_data_frame(base, table_name, record_data)) } @@ -377,7 +377,7 @@ air_make_json <- function (base, table_name, record_data, record_id = NULL, meth #' @param record_id String or vector of strings. Record id #' @param method String. One of "POST", "PATCH", or "DELETE" #' -#' @return +#' @return Status of HTTP request #' @export #' #' @examples @@ -429,9 +429,6 @@ air_make_request <- function(base, table_name, json_record_data, record_id = NUL Authorization = paste("Bearer", air_api_key()) ) ) - - air_validate(res) # throws exception (stop) if error - air_parse(res) # returns R object } air_validate(res) # throws exception (stop) if error diff --git a/man/air_make_json.Rd b/man/air_make_json.Rd new file mode 100644 index 0000000..b4ba714 --- /dev/null +++ b/man/air_make_json.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/airtabler.R +\name{air_make_json} +\alias{air_make_json} +\title{Make JSON for API} +\usage{ +air_make_json( + base, + table_name, + record_data, + record_id = NULL, + method = NULL, + typecast = TRUE +) +} +\arguments{ +\item{base}{String. Base in airtable} + +\item{table_name}{String. Table in airtable} + +\item{record_data}{Dataframe, list, or vector. Data to be converted to JSON} + +\item{record_id}{String or vector of strings. Records to be manipulated} + +\item{method}{String. "PATCH" is necessary for \code{air_update}} + +\item{typecast}{Logical. Should the typecast option be TRUE or FALSE? Typecast +allows you to add new options to select type fields.} +} +\value{ +JSON with record data +} +\description{ +Make JSON that is compatible with the Airtable API. +} diff --git a/man/air_make_request.Rd b/man/air_make_request.Rd new file mode 100644 index 0000000..d5f1257 --- /dev/null +++ b/man/air_make_request.Rd @@ -0,0 +1,31 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/airtabler.R +\name{air_make_request} +\alias{air_make_request} +\title{Make an HTTP request} +\usage{ +air_make_request( + base, + table_name, + json_record_data, + record_id = NULL, + method = c("POST", "PATCH", "DELETE") +) +} +\arguments{ +\item{base}{String. Base in airtable} + +\item{table_name}{String. Table in airtable} + +\item{json_record_data}{json or string. JSON formatted text with record data} + +\item{record_id}{String or vector of strings. Record id} + +\item{method}{String. One of "POST", "PATCH", or "DELETE"} +} +\value{ +Status of HTTP request +} +\description{ +Properly encodes HTTP requests +} From ecbb78c3207039733b51b18a03b9d461283fac14 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Wed, 6 Oct 2021 09:15:01 -0400 Subject: [PATCH 020/126] properly place typecast arguments for POST and PATCH so that new values can be passed to select type fields if desired. --- R/airtabler.R | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/R/airtabler.R b/R/airtabler.R index 2b876c3..6236c03 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -336,19 +336,13 @@ air_insert <- function(base, table_name, record_data) { #' @examples air_make_json <- function (base, table_name, record_data, record_id = NULL, method = "POST",typecast = TRUE){ if (inherits(record_data, "data.frame")) { - return(air_insert_data_frame(base, table_name, record_data)) + return(air_insert_data_frame(base, table_name, record_data, typecast)) } #browser() record_data <- air_prepare_record(as.list(record_data)) fields <- list(fields = record_data) - if(typecast){ - # allows us to use values not currently specified in the - # table or record eg issues = "some new issue" - # use unbox to create singleton values instead of json objects - fields$typecast <- jsonlite::unbox(typecast) - } if(method == "PATCH"){ ## the patch method already specifies a record so we @@ -357,11 +351,27 @@ air_make_json <- function (base, table_name, record_data, record_id = NULL, meth ## drop the id field as it can't be updated fields$fields$id <- NULL + if(typecast){ + # allows us to use values not currently specified in the + # table or record eg issues = "some new issue" + # use unbox to create singleton values instead of json objects + fields$typecast <- jsonlite::unbox(typecast) + } + + json_patch_data <- jsonlite::toJSON(fields, pretty = T) return(json_patch_data) } records <- list(records = list(fields)) + + if(typecast){ + # allows us to use values not currently specified in the + # table or record eg issues = "some new issue" + # use unbox to create singleton values instead of json objects + records$typecast <- jsonlite::unbox(typecast) + } + json_record_data <- jsonlite::toJSON(records, pretty = T) return(json_record_data) } @@ -441,15 +451,27 @@ air_make_request <- function(base, table_name, json_record_data, record_id = NUL #' #' @rdname air_insert #' @export air_insert_data_frame -air_insert_data_frame <- function(base, table_name, records) { +air_insert_data_frame <- function(base, table_name, records,typecast) { lapply(seq_len(nrow(records)), function(i) { record_data <- as.list(records[i,]) - json_record_data <- air_make_json(base, table_name, record_data) + json_record_data <- air_make_json(base, table_name, record_data,typecast) air_make_request(base = base,table_name = table_name ,json_record_data = json_record_data, method = "POST" ) }) } +#' Update records from a dataframe +#' +#' +#' @param base String. Airtable base +#' @param table_name String. Table name +#' @param record_ids Vector of strings. Records to be modified +#' @param records Dataframe. Values to update +#' +#' @return +#' @export +#' +#' @examples air_update_data_frame <- function(base, table_name, record_ids, records) { lapply(seq_len(nrow(records)), function(i) { record_data <- as.list(records[i,]) From ca491080dfbf6c3127091a65e05d8f28ca654bbc Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 22 Oct 2021 14:07:44 -0400 Subject: [PATCH 021/126] Update readme.md --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 522506a..301d081 100644 --- a/readme.md +++ b/readme.md @@ -7,7 +7,7 @@ Provides access to the [Airtable API](http://airtable.com/api) ```r -devtools::install_github("bergant/airtabler") +devtools::install_github("ecohealthalliance/airtabler") ``` ## Setup From 98f6d7e9b1a15ad813dc4317289274dd06b1de2c Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 22 Oct 2021 14:12:37 -0400 Subject: [PATCH 022/126] added a download feature to air_get_attachments --- R/air_get_attachments.R | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/R/air_get_attachments.R b/R/air_get_attachments.R index 8c886f8..72239a7 100644 --- a/R/air_get_attachments.R +++ b/R/air_get_attachments.R @@ -9,13 +9,15 @@ #' Should be one of: excel #' @param extract_field String. Name of extract field that will be created #' @param ... Additional arguments to pass to \code{air_get} -#' +#' @param download_file Logical. Should files be downloaded? +#' @param dir_name String. Where should files be downloaded to? +#' Will create the folder if it does not exist. #' #' @return named list of data frames #' @export air_get_attachments #' #' @examples -air_get_attachments <- function(base, table_name, field, extract_type ="excel", extract_field ="excel_extract", ...){ +air_get_attachments <- function(base, table_name, field, download_file = FALSE, dir_name = "downloads", extract_type ="excel", extract_field ="excel_extract", ...){ #browser() # get data x <- fetch_all(base,table_name,...) @@ -26,6 +28,29 @@ air_get_attachments <- function(base, table_name, field, extract_type ="excel", xfield <- purrr::pluck(x,field) ### get files ---- + + if(download_file){ + dir.create(dir_name) + + xlist <- purrr::map(xfield, function(x){ + + if(is.null(x$url)){ + ID <- x$id + warning(sprintf("Record ID %s is null",ID)) + return(NULL) + } + + destfile <- sprintf("%s/%s",dir_name ,basename(x$url)) + + download.file(url = x$url,destfile = destfile) + }) + + message("downloaded files in ./downloads") + + } + + ### extract excel ---- + if(extract_type == "excel"){ xlist <- purrr::map(xfield,function(x){ From e7911d5f5f1561123f4f3a1292c687cce952cd62 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 22 Oct 2021 14:25:22 -0400 Subject: [PATCH 023/126] fixed minor issues in documentation and function calls --- DESCRIPTION | 3 ++- NAMESPACE | 1 + R/air_get_attachments.R | 2 +- R/airtabler.R | 2 ++ man/air_get_attachments.Rd | 7 +++++++ man/air_insert.Rd | 4 +++- man/air_make_json.Rd | 2 +- man/air_update_data_frame.Rd | 23 +++++++++++++++++++++++ 8 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 man/air_update_data_frame.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 5f14e43..f11ddfd 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -18,5 +18,6 @@ Imports: jsonlite, readxl, curl, - purrr + purrr, + utils RoxygenNote: 7.1.1 diff --git a/NAMESPACE b/NAMESPACE index 2544008..8c79221 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -11,6 +11,7 @@ export(air_make_json) export(air_make_request) export(air_select) export(air_update) +export(air_update_data_frame) export(airtable) export(fetch_all) export(get_offset) diff --git a/R/air_get_attachments.R b/R/air_get_attachments.R index 72239a7..1deabdb 100644 --- a/R/air_get_attachments.R +++ b/R/air_get_attachments.R @@ -42,7 +42,7 @@ air_get_attachments <- function(base, table_name, field, download_file = FALSE, destfile <- sprintf("%s/%s",dir_name ,basename(x$url)) - download.file(url = x$url,destfile = destfile) + utils::download.file(url = x$url,destfile = destfile) }) message("downloaded files in ./downloads") diff --git a/R/airtabler.R b/R/airtabler.R index 6236c03..c4e1725 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -448,6 +448,7 @@ air_make_request <- function(base, table_name, json_record_data, record_id = NUL #' @param base String. Airtable base #' @param table_name String. Table name #' @param records Dataframe. Contains records you would like to insert +#' @param typecast Logical. Should airtable make new values for select type fields? #' #' @rdname air_insert #' @export air_insert_data_frame @@ -462,6 +463,7 @@ air_insert_data_frame <- function(base, table_name, records,typecast) { #' Update records from a dataframe #' +#' Updates the values in a table by overwriting their current contents. #' #' @param base String. Airtable base #' @param table_name String. Table name diff --git a/man/air_get_attachments.Rd b/man/air_get_attachments.Rd index 7314c60..cf9355e 100644 --- a/man/air_get_attachments.Rd +++ b/man/air_get_attachments.Rd @@ -8,6 +8,8 @@ air_get_attachments( base, table_name, field, + download_file = FALSE, + dir_name = "downloads", extract_type = "excel", extract_field = "excel_extract", ... @@ -20,6 +22,11 @@ air_get_attachments( \item{field}{String. Name of field with file attachments in base} +\item{download_file}{Logical. Should files be downloaded?} + +\item{dir_name}{String. Where should files be downloaded to? +Will create the folder if it does not exist.} + \item{extract_type}{String. File type to be extracted. Should be one of: excel} diff --git a/man/air_insert.Rd b/man/air_insert.Rd index 20cfd59..9c605c8 100644 --- a/man/air_insert.Rd +++ b/man/air_insert.Rd @@ -8,7 +8,7 @@ \usage{ air_insert(base, table_name, record_data) -air_insert_data_frame(base, table_name, records) +air_insert_data_frame(base, table_name, records, typecast) multiple(x) } @@ -22,6 +22,8 @@ of the field values} \item{records}{Dataframe. Contains records you would like to insert} +\item{typecast}{Logical. Should airtable make new values for select type fields?} + \item{x}{Object to be marked as a multiple value field} } \description{ diff --git a/man/air_make_json.Rd b/man/air_make_json.Rd index b4ba714..541c47a 100644 --- a/man/air_make_json.Rd +++ b/man/air_make_json.Rd @@ -9,7 +9,7 @@ air_make_json( table_name, record_data, record_id = NULL, - method = NULL, + method = "POST", typecast = TRUE ) } diff --git a/man/air_update_data_frame.Rd b/man/air_update_data_frame.Rd new file mode 100644 index 0000000..db23c37 --- /dev/null +++ b/man/air_update_data_frame.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/airtabler.R +\name{air_update_data_frame} +\alias{air_update_data_frame} +\title{Update records from a dataframe} +\usage{ +air_update_data_frame(base, table_name, record_ids, records) +} +\arguments{ +\item{base}{String. Airtable base} + +\item{table_name}{String. Table name} + +\item{record_ids}{Vector of strings. Records to be modified} + +\item{records}{Dataframe. Values to update} +} +\value{ + +} +\description{ +Updates the values in a table by overwriting their current contents. +} From 740718fbbcc5e0df73fdb58719b4511db82ea0c7 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 29 Oct 2021 14:12:28 -0400 Subject: [PATCH 024/126] changed the way file names are constructed to use the filename attribute instead of parsing the URL for the file. --- R/air_get_attachments.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/air_get_attachments.R b/R/air_get_attachments.R index 1deabdb..a5f8cde 100644 --- a/R/air_get_attachments.R +++ b/R/air_get_attachments.R @@ -40,9 +40,9 @@ air_get_attachments <- function(base, table_name, field, download_file = FALSE, return(NULL) } - destfile <- sprintf("%s/%s",dir_name ,basename(x$url)) + dest <- sprintf("%s/%s", dir_name,x$filename) - utils::download.file(url = x$url,destfile = destfile) + utils::download.file(url = x$url,destfile = dest) }) message("downloaded files in ./downloads") From 30d9fa3137a84ed2ac8a3ccef30ccdc54f4eca70 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 24 Jun 2022 10:40:36 -0600 Subject: [PATCH 025/126] Update readme.rmd --- readme.rmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.rmd b/readme.rmd index 9f6ddeb..4d6a6d3 100644 --- a/readme.rmd +++ b/readme.rmd @@ -13,7 +13,7 @@ knitr::opts_chunk$set(echo = TRUE) ## Install ```{r install, eval=FALSE} -devtools::install_github("bergant/airtabler") +devtools::install_github("ecohealthalliance/airtabler") ``` ## Setup From 41e8a056fdd765c92ab71892f00ff70f0050434e Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 27 Jun 2022 12:10:19 -0500 Subject: [PATCH 026/126] added air_dump function --- R/air_dump.R | 137 ++++++++++++++++++++++++++++++++++++++++++++++++ man/air_dump.Rd | 45 ++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 R/air_dump.R create mode 100644 man/air_dump.Rd diff --git a/R/air_dump.R b/R/air_dump.R new file mode 100644 index 0000000..5d181ad --- /dev/null +++ b/R/air_dump.R @@ -0,0 +1,137 @@ +#' Output tables to a specified file format +#' +#' This function uses metadata tables in your base to extract all +#' base table to a different file format. Currently only CSV is supported. +#' +#' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' +#' @param table_name_metadata String. Name of structural metadata table. PROVIDE LINK TO DEF OF STRUCTURAL METADATA +#' @param table_name_description String. Name of the descriptive metadata table. PROVIDE LINK TO DEF OF DESCRIPTIVE METADATA +#' @param output_dir String. Folder containing outputs. If not specified, a folder +#' with the name of the output_id will be used. +#' @param output_format String. Default is csv. Options include: csv +#' @param to_frictionless_data Logical. Requires table_name_metadata and table_name_description. +#' Should data be output as a frictionless data package. (To be Added) +#' @param all_table_names Vector. Do not use if providing table_name_metadata. +#' A character vector of all table names in your base. I.e c("table 1","table 2"). +#' @param output_id String. ID used to uniquely identify outputs. A folder with +#' this name will be created in output_dir. +#' +#' @return List of file paths created. +#' @export +#' +#' @examples + +air_dump <- function(base = Sys.getenv("airtable_metadata_base"), + table_name_metadata = "meta data", + table_name_description = "description", + all_table_names = NULL, + output_dir = "outputs", + output_id = rlang::hash(Sys.time()), + output_format = "csv", + to_frictionless_data=FALSE){ + + if(is.null(output_dir)){ + output_dir <- "." + message(glue::glue("No ouput_dir provided. + Files are being saved directly to ./{output_id}")) + } + + temp_path <- tempdir() + ## create dir - should probably go to a temp dir until finished + output_dir_path <- sprintf("%s/%s",temp_path,output_id) + dir.create(path = output_dir_path,recursive = T, showWarnings = FALSE) + + ## check that output_format is supported + output_format <- tolower(output_format) + output_formats <- c("csv") # potentially include json, txt,xml + if(!any(output_format %in% output_formats)){ + stop(glue::glue("{output_format} not supported. Use one of {output_formats}")) + } + + ## load metadata table - limit to just table names + ### create structural metadata file + + # write structural metadata table + str_metadata <- airtabler::fetch_all(base,table_name_metadata) + smf_file_path <- sprintf("%s/%s.csv",output_dir_path,snakecase::to_lower_camel_case(table_name_metadata)) + utils::write.csv(str_metadata,smf_file_path,row.names = FALSE) + + base_table_names <- unique(str_metadata$`Table Name`) + base_table_names_formatted <- snakecase::to_lower_camel_case(base_table_names) + purrr::map2(base_table_names,base_table_names_formatted, function(x,y){ + #browser() + ## get fields from str_metadata + + fields_exp <- str_metadata[str_metadata$`Table Name` == x,"Field Name"] + + ## pull table - add check for blank tables + x_table <- airtabler::fetch_all(base,x) + + if(!is.data.frame(x_table)){ + x_table <- data.frame(id = character()) + } + + ## add in missing columns + fields_obs <- names(x_table) + # drop autogenerated cols from comparison + fields_obs <- fields_obs[!grepl(pattern = "^id$|^createdTime$", + x = fields_obs)] + # check if any discrepancy between metadata and table + fields_diff <- set_diff(fields_exp,fields_obs) + + if(!is.null(fields_diff)){ + # check for fields in obs not in exp - error + obs_exp <- setdiff(fields_obs,fields_exp) + if(length(obs_exp) != 0){ + stop(glue::glue("Table {x} contains field(s) {obs_exp} not found in {table_name_metadata}. Update metadata table.")) + } + # check for fields in exp and not in obs - append unless frictionless + if(!to_frictionless_data){ + exp_obs <- setdiff(fields_exp,fields_obs) + x_table[exp_obs] <- list(character(0)) + } + } + + ## clean up field names + + names(x_table) <- snakecase::to_lower_camel_case(names(x_table)) + + # convert list type fields to strings + + x_table_flat <- dplyr::mutate(.data = x_table, + dplyr::across(tidyselect:::where(is.list), purrr::flatten_chr)) + + ## export to CSV + + output_file_path <- sprintf("%s/%s.csv",output_dir_path,y) + + utils::write.csv(x_table_flat,output_file_path,row.names = FALSE) + }) + + ## copy files from temp to final + + output_dir_path_final <- sprintf("%s/%s",output_dir,output_id) + dir.create(output_dir_path_final) + + outputs_list <- list.files(output_dir_path,full.names = T) + + file.copy(from = outputs_list,to = output_dir_path_final,recursive = FALSE ,copy.mode = TRUE) + + message(glue::glue("Files can be found here: {output_dir_path_final}")) + + return(list.files(output_dir_path_final,full.names = T)) + +} + +set_diff <- function(x,y){ + u <- union(x,y) + i <- intersect(x,y) + j <- (u %in% i) + + if(all(j)){ + return(NULL) + } + + diff <- u[!(j)] + return(diff) +} diff --git a/man/air_dump.Rd b/man/air_dump.Rd new file mode 100644 index 0000000..3f0b3c4 --- /dev/null +++ b/man/air_dump.Rd @@ -0,0 +1,45 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_dump} +\alias{air_dump} +\title{Output tables to a specified file format} +\usage{ +air_dump( + base = Sys.getenv("airtable_metadata_base"), + table_name_metadata = "meta data", + table_name_description = "description", + all_table_names = NULL, + output_dir = "outputs", + output_id = rlang::hash(Sys.time()), + output_format = "csv", + to_frictionless_data = FALSE +) +} +\arguments{ +\item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} + +\item{table_name_metadata}{String. Name of structural metadata table. PROVIDE LINK TO DEF OF STRUCTURAL METADATA} + +\item{table_name_description}{String. Name of the descriptive metadata table. PROVIDE LINK TO DEF OF DESCRIPTIVE METADATA} + +\item{all_table_names}{Vector. Do not use if providing table_name_metadata. +A character vector of all table names in your base. I.e c("table 1","table 2").} + +\item{output_dir}{String. Folder containing outputs. If not specified, a folder +with the name of the output_id will be used.} + +\item{output_id}{String. ID used to uniquely identify outputs. A folder with +this name will be created in output_dir.} + +\item{output_format}{String. Default is csv. Options include: To be added} + +\item{to_frictionless_data}{Logical. Requires table_name_metadata and table_name_description. +Should data be output as a frictionless data package. (To be Added)} +} +\value{ +List of file paths created. +} +\description{ +This function uses metadata tables in your base to extract all +base table to a different file format. Currently only CSV is supported. +} From d36033619ca31e4f7621675686396dc55439d84a Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 27 Jun 2022 12:11:00 -0500 Subject: [PATCH 027/126] adding air_dump to package desc and namespace --- DESCRIPTION | 6 ++++-- NAMESPACE | 1 + man/air_update_data_frame.Rd | 3 --- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index f11ddfd..c39d8d3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -19,5 +19,7 @@ Imports: readxl, curl, purrr, - utils -RoxygenNote: 7.1.1 + utils, + snakecase, + tidyselect +RoxygenNote: 7.2.0 diff --git a/NAMESPACE b/NAMESPACE index 8c79221..d6c49bc 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -2,6 +2,7 @@ S3method(print,airtable.base) export(air_delete) +export(air_dump) export(air_get) export(air_get_attachments) export(air_get_schema) diff --git a/man/air_update_data_frame.Rd b/man/air_update_data_frame.Rd index db23c37..375cd71 100644 --- a/man/air_update_data_frame.Rd +++ b/man/air_update_data_frame.Rd @@ -14,9 +14,6 @@ air_update_data_frame(base, table_name, record_ids, records) \item{record_ids}{Vector of strings. Records to be modified} \item{records}{Dataframe. Values to update} -} -\value{ - } \description{ Updates the values in a table by overwriting their current contents. From e2d9f6be7bab67cba3b3af7e5d5e51f3caa5f5c4 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 27 Jun 2022 12:15:45 -0500 Subject: [PATCH 028/126] using snake_case in stead of lower camel for consistency with package --- R/air_dump.R | 6 +++--- man/air_dump.Rd | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index 5d181ad..adc840e 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -53,11 +53,11 @@ air_dump <- function(base = Sys.getenv("airtable_metadata_base"), # write structural metadata table str_metadata <- airtabler::fetch_all(base,table_name_metadata) - smf_file_path <- sprintf("%s/%s.csv",output_dir_path,snakecase::to_lower_camel_case(table_name_metadata)) + smf_file_path <- sprintf("%s/%s.csv",output_dir_path,snakecase::to_snake_case(table_name_metadata)) utils::write.csv(str_metadata,smf_file_path,row.names = FALSE) base_table_names <- unique(str_metadata$`Table Name`) - base_table_names_formatted <- snakecase::to_lower_camel_case(base_table_names) + base_table_names_formatted <- snakecase::to_snake_case(base_table_names) purrr::map2(base_table_names,base_table_names_formatted, function(x,y){ #browser() ## get fields from str_metadata @@ -94,7 +94,7 @@ air_dump <- function(base = Sys.getenv("airtable_metadata_base"), ## clean up field names - names(x_table) <- snakecase::to_lower_camel_case(names(x_table)) + names(x_table) <- snakecase::to_snake_case(names(x_table)) # convert list type fields to strings diff --git a/man/air_dump.Rd b/man/air_dump.Rd index 3f0b3c4..4f38570 100644 --- a/man/air_dump.Rd +++ b/man/air_dump.Rd @@ -31,7 +31,7 @@ with the name of the output_id will be used.} \item{output_id}{String. ID used to uniquely identify outputs. A folder with this name will be created in output_dir.} -\item{output_format}{String. Default is csv. Options include: To be added} +\item{output_format}{String. Default is csv. Options include: csv} \item{to_frictionless_data}{Logical. Requires table_name_metadata and table_name_description. Should data be output as a frictionless data package. (To be Added)} From f0c4e97c9aa63d95872b5dd0563bd812fe4aa7fc Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 27 Jun 2022 12:38:44 -0500 Subject: [PATCH 029/126] add recursive argument to create dir --- R/air_dump.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/air_dump.R b/R/air_dump.R index adc840e..9554eff 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -54,6 +54,7 @@ air_dump <- function(base = Sys.getenv("airtable_metadata_base"), # write structural metadata table str_metadata <- airtabler::fetch_all(base,table_name_metadata) smf_file_path <- sprintf("%s/%s.csv",output_dir_path,snakecase::to_snake_case(table_name_metadata)) + # names(str_metadata) <- snakecase::to_lower_camel_case(names(str_metadata)) utils::write.csv(str_metadata,smf_file_path,row.names = FALSE) base_table_names <- unique(str_metadata$`Table Name`) @@ -111,7 +112,7 @@ air_dump <- function(base = Sys.getenv("airtable_metadata_base"), ## copy files from temp to final output_dir_path_final <- sprintf("%s/%s",output_dir,output_id) - dir.create(output_dir_path_final) + dir.create(output_dir_path_final,recursive = T) outputs_list <- list.files(output_dir_path,full.names = T) From 68ce7f99590efecac6bb8f42959c5e136b0201d2 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 27 Jun 2022 12:46:15 -0500 Subject: [PATCH 030/126] added documentation for set_diff --- NAMESPACE | 1 + R/air_dump.R | 25 +++++++++++++++++++++++++ man/set_diff.Rd | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 man/set_diff.Rd diff --git a/NAMESPACE b/NAMESPACE index d6c49bc..0bfc9db 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -19,3 +19,4 @@ export(get_offset) export(get_unique_field_values) export(multiple) export(read_excel_url) +export(set_diff) diff --git a/R/air_dump.R b/R/air_dump.R index 9554eff..b2b4cf7 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -124,6 +124,31 @@ air_dump <- function(base = Sys.getenv("airtable_metadata_base"), } +#' Get items that differ between x and y +#' +#' Unlike setdiff, this function creates the union of x and y then +#' removes values that are in the intersect, providing values +#' that are unique to X and values that are unique to Y. +#' +#' @param x a set of values. +#' @param y a set of values. +#' +#' @return Unique values from X and Y, NULL if no unique values. +#' @export +#' +#' @examples +#' a <- 1:3 +#' b <- 2:4 +#' +#' set_diff(a,b) +#' # returns 1,4 +#' +#' x <- 1:3 +#' y <- 1:3 +#' +#' set_diff(x,y) +#' # returns NULL +#' set_diff <- function(x,y){ u <- union(x,y) i <- intersect(x,y) diff --git a/man/set_diff.Rd b/man/set_diff.Rd new file mode 100644 index 0000000..8d71da6 --- /dev/null +++ b/man/set_diff.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{set_diff} +\alias{set_diff} +\title{Get items that differ between x and y} +\usage{ +set_diff(x, y) +} +\arguments{ +\item{x}{a set of values.} + +\item{y}{a set of values.} +} +\value{ +Unique values from X and Y, NULL if no unique values. +} +\description{ +Unlike setdiff, this function creates the union of x and y then +removes values that are in the intersect, providing values +that are unique to X and values that are unique to Y. +} +\examples{ +a <- 1:3 +b <- 2:4 + +set_diff(a,b) +# returns 1,4 + +x <- 1:3 +y <- 1:3 + +set_diff(x,y) +# returns NULL + +} From fce4fea9dc3d78d14cffffbf578f5f98be686b1a Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Wed, 29 Jun 2022 10:40:39 -0500 Subject: [PATCH 031/126] added note on where to find help for metadata tables --- R/air_dump.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/R/air_dump.R b/R/air_dump.R index b2b4cf7..08428cb 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -2,6 +2,8 @@ #' #' This function uses metadata tables in your base to extract all #' base table to a different file format. Currently only CSV is supported. +#' For information about creating metadata tables in your base see the +#' \href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} #' #' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' #' @param table_name_metadata String. Name of structural metadata table. PROVIDE LINK TO DEF OF STRUCTURAL METADATA @@ -161,3 +163,4 @@ set_diff <- function(x,y){ diff <- u[!(j)] return(diff) } + From 19b04524dc90a6060a7372927379aa75ac5665a5 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Wed, 29 Jun 2022 10:41:06 -0500 Subject: [PATCH 032/126] added note on account requirements for metadata api --- R/air_get_schema.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/air_get_schema.R b/R/air_get_schema.R index 46e19be..1c13fe2 100644 --- a/R/air_get_schema.R +++ b/R/air_get_schema.R @@ -1,5 +1,6 @@ -#' Get base schema +#' Get base schema - enterprise only #' +#' Metadata API currently only available via enterprise accounts. #' Get the schema for the tables in a base. #' #' @section Using Metadata API: From c24d40965548c28e97c247a93ceacfcae1af41a4 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 30 Jun 2022 09:17:56 -0500 Subject: [PATCH 033/126] breaking air_dump into component parts --- R/air_dump.R | 390 +++++++++++++++++++++++++++++++++++------------- man/air_dump.Rd | 2 + 2 files changed, 290 insertions(+), 102 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index 08428cb..74c772f 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -23,108 +23,108 @@ #' #' @examples -air_dump <- function(base = Sys.getenv("airtable_metadata_base"), - table_name_metadata = "meta data", - table_name_description = "description", - all_table_names = NULL, - output_dir = "outputs", - output_id = rlang::hash(Sys.time()), - output_format = "csv", - to_frictionless_data=FALSE){ - - if(is.null(output_dir)){ - output_dir <- "." - message(glue::glue("No ouput_dir provided. - Files are being saved directly to ./{output_id}")) - } - - temp_path <- tempdir() - ## create dir - should probably go to a temp dir until finished - output_dir_path <- sprintf("%s/%s",temp_path,output_id) - dir.create(path = output_dir_path,recursive = T, showWarnings = FALSE) - - ## check that output_format is supported - output_format <- tolower(output_format) - output_formats <- c("csv") # potentially include json, txt,xml - if(!any(output_format %in% output_formats)){ - stop(glue::glue("{output_format} not supported. Use one of {output_formats}")) - } - - ## load metadata table - limit to just table names - ### create structural metadata file - - # write structural metadata table - str_metadata <- airtabler::fetch_all(base,table_name_metadata) - smf_file_path <- sprintf("%s/%s.csv",output_dir_path,snakecase::to_snake_case(table_name_metadata)) - # names(str_metadata) <- snakecase::to_lower_camel_case(names(str_metadata)) - utils::write.csv(str_metadata,smf_file_path,row.names = FALSE) - - base_table_names <- unique(str_metadata$`Table Name`) - base_table_names_formatted <- snakecase::to_snake_case(base_table_names) - purrr::map2(base_table_names,base_table_names_formatted, function(x,y){ - #browser() - ## get fields from str_metadata - - fields_exp <- str_metadata[str_metadata$`Table Name` == x,"Field Name"] - - ## pull table - add check for blank tables - x_table <- airtabler::fetch_all(base,x) - - if(!is.data.frame(x_table)){ - x_table <- data.frame(id = character()) - } - - ## add in missing columns - fields_obs <- names(x_table) - # drop autogenerated cols from comparison - fields_obs <- fields_obs[!grepl(pattern = "^id$|^createdTime$", - x = fields_obs)] - # check if any discrepancy between metadata and table - fields_diff <- set_diff(fields_exp,fields_obs) - - if(!is.null(fields_diff)){ - # check for fields in obs not in exp - error - obs_exp <- setdiff(fields_obs,fields_exp) - if(length(obs_exp) != 0){ - stop(glue::glue("Table {x} contains field(s) {obs_exp} not found in {table_name_metadata}. Update metadata table.")) - } - # check for fields in exp and not in obs - append unless frictionless - if(!to_frictionless_data){ - exp_obs <- setdiff(fields_exp,fields_obs) - x_table[exp_obs] <- list(character(0)) - } - } - - ## clean up field names - - names(x_table) <- snakecase::to_snake_case(names(x_table)) - - # convert list type fields to strings - - x_table_flat <- dplyr::mutate(.data = x_table, - dplyr::across(tidyselect:::where(is.list), purrr::flatten_chr)) - - ## export to CSV - - output_file_path <- sprintf("%s/%s.csv",output_dir_path,y) - - utils::write.csv(x_table_flat,output_file_path,row.names = FALSE) - }) - - ## copy files from temp to final - - output_dir_path_final <- sprintf("%s/%s",output_dir,output_id) - dir.create(output_dir_path_final,recursive = T) - - outputs_list <- list.files(output_dir_path,full.names = T) - - file.copy(from = outputs_list,to = output_dir_path_final,recursive = FALSE ,copy.mode = TRUE) - - message(glue::glue("Files can be found here: {output_dir_path_final}")) - - return(list.files(output_dir_path_final,full.names = T)) - -} +# air_dump <- function(base = Sys.getenv("airtable_metadata_base"), +# table_name_metadata = "meta data", +# table_name_description = "description", +# all_table_names = NULL, +# output_dir = "outputs", +# output_id = rlang::hash(Sys.time()), +# output_format = "csv", +# to_frictionless_data=FALSE){ +# +# if(is.null(output_dir)){ +# output_dir <- "." +# message(glue::glue("No ouput_dir provided. +# Files are being saved directly to ./{output_id}")) +# } +# +# temp_path <- tempdir() +# ## create dir - should probably go to a temp dir until finished +# output_dir_path <- sprintf("%s/%s",temp_path,output_id) +# dir.create(path = output_dir_path,recursive = T, showWarnings = FALSE) +# +# ## check that output_format is supported +# output_format <- tolower(output_format) +# output_formats <- c("csv") # potentially include json, txt,xml +# if(!any(output_format %in% output_formats)){ +# stop(glue::glue("{output_format} not supported. Use one of {output_formats}")) +# } +# +# ## load metadata table - limit to just table names +# ### create structural metadata file +# +# # write structural metadata table +# str_metadata <- airtabler::fetch_all(base,table_name_metadata) +# smf_file_path <- sprintf("%s/%s.csv",output_dir_path,snakecase::to_snake_case(table_name_metadata)) +# # names(str_metadata) <- snakecase::to_lower_camel_case(names(str_metadata)) +# utils::write.csv(str_metadata,smf_file_path,row.names = FALSE) +# +# base_table_names <- unique(str_metadata$`Table Name`) +# base_table_names_formatted <- snakecase::to_snake_case(base_table_names) +# purrr::map2(base_table_names,base_table_names_formatted, function(x,y){ +# #browser() +# ## get fields from str_metadata +# +# fields_exp <- str_metadata[str_metadata$`Table Name` == x,"Field Name"] +# +# ## pull table - add check for blank tables +# x_table <- airtabler::fetch_all(base,x) +# +# if(!is.data.frame(x_table)){ +# x_table <- data.frame(id = character()) +# } +# +# ## add in missing columns +# fields_obs <- names(x_table) +# # drop autogenerated cols from comparison +# fields_obs <- fields_obs[!grepl(pattern = "^id$|^createdTime$", +# x = fields_obs)] +# # check if any discrepancy between metadata and table +# fields_diff <- set_diff(fields_exp,fields_obs) +# +# if(!is.null(fields_diff)){ +# # check for fields in obs not in exp - error +# obs_exp <- setdiff(fields_obs,fields_exp) +# if(length(obs_exp) != 0){ +# stop(glue::glue("Table {x} contains field(s) {obs_exp} not found in {table_name_metadata}. Update metadata table.")) +# } +# # check for fields in exp and not in obs - append unless frictionless +# if(!to_frictionless_data){ +# exp_obs <- setdiff(fields_exp,fields_obs) +# x_table[exp_obs] <- list(character(0)) +# } +# } +# +# ## clean up field names +# +# names(x_table) <- snakecase::to_snake_case(names(x_table)) +# +# # convert list type fields to strings +# +# x_table_flat <- dplyr::mutate(.data = x_table, +# dplyr::across(tidyselect:::where(is.list), purrr::flatten_chr)) +# +# ## export to CSV +# +# output_file_path <- sprintf("%s/%s.csv",output_dir_path,y) +# +# utils::write.csv(x_table_flat,output_file_path,row.names = FALSE) +# }) +# +# ## copy files from temp to final +# +# output_dir_path_final <- sprintf("%s/%s",output_dir,output_id) +# dir.create(output_dir_path_final,recursive = T) +# +# outputs_list <- list.files(output_dir_path,full.names = T) +# +# file.copy(from = outputs_list,to = output_dir_path_final,recursive = FALSE ,copy.mode = TRUE) +# +# message(glue::glue("Files can be found here: {output_dir_path_final}")) +# +# return(list.files(output_dir_path_final,full.names = T)) +# +# } #' Get items that differ between x and y #' @@ -164,3 +164,189 @@ set_diff <- function(x,y){ return(diff) } +### get_metadata - returns a data frame + +air_get_metadata_from_table <- function(base, table_name){ + # get structural metadata table + str_metadata <- airtabler::fetch_all(base,table_name) + ## check for table_name, field_name + names(str_metadata) <- snakecase::to_snake_case(names(str_metadata)) + + required_fields <- c("table_name","field_name") + if(!all(required_fields %in% names(str_metadata))){ + stop(glue::glue("metadata table must contain the + following fields: {required_fields}. Note + that field names are converted to snakecase + before check.")) + } + + return(str_metadata) +} + +### generate metadata - returns a data frame +air_generate_metadata <- function(base, table_names,limit=1){ + warning('Airtable does not return fields with empty values - "", false, or []. + It is better to create a specific metdata table and + parse that with air_get_metadata_*') + meta_data_table <- purrr::map_dfr(table_names,function(x){ + table_x <- airtabler::air_get(base,x,limit = limit ) + fields_x <- names(table_x) + + ## guess record types? + + md_df <- data.frame(table_name = x, field_name = fields_x, field_desc = "", field_type = "") + + return(md_df) + }) + + return(meta_data_table) +} + +air_get_base_description_from_table<- function(base, table_name){ + #fetch table + desc_table <- airtabler::fetch_all(base,table_name) + # to snake case + names(desc_table) <- snakecase::to_snake_case(names(desc_table)) + + required_fields <- c("title","primary_contact","email","base_description") + if(all(required_fields %in% names(desc_table))){ + return(desc_table) + } else { + + missing_rf <- required_fields[!required_fields %in% names(desc_table)] + + desc_table[missing_rf] <- NA + return(desc_table) + } + +} + +air_generate_base_description <- function(title = NA,primary_contact= NA,email = NA, base_description = NA,...){ + desc_table <- data.frame(title,primary_contact,email,base_description,...) + return(desc_table) +} + +### extract_base - returns a named list + +air_dump <- function(base, metadata, description = NULL, add_missing_fields = TRUE){ + + names(metadata) <- snakecase::to_snake_case(names(metadata)) + ## check for required fields + required_fields <- c("table_name","field_name") + + if(!all(required_fields %in% names(metadata))){ + stop(glue::glue("metadata table must contain the + following fields: {required_fields}. Note + that field names are converted to snakecase + before check.")) + } + + + base_table_names <- unique(metadata$table_name) + + table_list <- base_table_names |> + purrr::set_names() |> + purrr::map(function(x){ + #browser() + ## get fields from str_metadata + + fields_exp <- metadata[metadata$table_name == x,"field_name"] + + ## pull table - add check for blank tables + x_table <- airtabler::fetch_all(base,x) + + if(!is.data.frame(x_table)){ + x_table <- data.frame(id = character()) + } + + ## add in missing columns if any + fields_obs <- names(x_table) + + # check if any discrepancy between metadata and table + fields_diff <- set_diff(fields_exp,fields_obs) + + if(!is.null(fields_diff)){ + # check for fields in obs not in exp - error + obs_exp <- setdiff(fields_obs,fields_exp) + if(length(obs_exp) != 0 & !all(obs_exp %in% c("id","createdTime"))){ + stop(glue::glue("Table {x} contains field(s) {obs_exp} not found in {table_name_metadata}. Update metadata table.")) + } + # check for fields in exp and not in obs - append unless frictionless + if(add_missing_fields){ + exp_obs <- setdiff(fields_exp,fields_obs) + x_table[exp_obs] <- list(character(0)) + } + } + + return(x_table) + + }) + + table_list$metadata <- metadata + + if(!is.null(description)){ + table_list$description <- description + } else { + ## give null description + table_list$description <- air_generate_base_description() + } + + table_list$description$date_created <- Sys.Date() + + return(table_list) +} + + +### write to X + +air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE){ + + output_id <- rlang::hash(table_list) + output_dir_path <- sprintf("%s/%s",output_dir,output_id) + # check if data already exist + + if(dir.exists(output_dir_path) & !overwrite){ + message("data already exist, files not written") + return(list.files(output_dir_path)) + } + + dir.create(output_dir_path,recursive = TRUE) + + purrr::walk2(table_list, names(table_list), function(x_table,y_table_name){ + ## clean table name + y_table_name <- snakecase::to_snake_case(y_table_name) + ## clean up field names in table + names(x_table) <- snakecase::to_snake_case(names(x_table)) + + + ## clean up field names + names(x_table) <- snakecase::to_snake_case(names(x_table)) + if(y_table_name == "table_2"){ + browser() + } + + purrr::map_dfc(x_table, function(x){ + if(is.list(x)){ + x <- purrr::flatten_dfc(x) + } + return(x) + }) + + # convert list type fields to strings + x_table_flat <- dplyr::mutate(.data = x_table, + dplyr::across( + tidyselect:::where(is.list), + unlist) + ) + + ## export to CSV + + output_file_path <- sprintf("%s/%s.csv",output_dir_path,y_table_name) + + utils::write.csv(x_table_flat,output_file_path,row.names = FALSE) + }) +} + + + +### recover from metadata - JS code to regenerate tables diff --git a/man/air_dump.Rd b/man/air_dump.Rd index 4f38570..5aeab14 100644 --- a/man/air_dump.Rd +++ b/man/air_dump.Rd @@ -42,4 +42,6 @@ List of file paths created. \description{ This function uses metadata tables in your base to extract all base table to a different file format. Currently only CSV is supported. +For information about creating metadata tables in your base see the +\href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} } From 513ae1a8499ed05404f8cd7a4da7ce70baf301c2 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 30 Jun 2022 15:28:17 -0500 Subject: [PATCH 034/126] added documentation for air_dump functions --- DESCRIPTION | 5 +- NAMESPACE | 6 + R/air_dump.R | 346 ++++++++++++--------- man/air_dump.Rd | 40 +-- man/air_dump_to_csv.Rd | 24 ++ man/air_generate_base_description.Rd | 54 ++++ man/air_generate_metadata.Rd | 29 ++ man/air_get_base_description_from_table.Rd | 22 ++ man/air_get_metadata_from_table.Rd | 24 ++ man/air_get_schema.Rd | 3 +- man/flatten_col_to_chr.Rd | 41 +++ 11 files changed, 408 insertions(+), 186 deletions(-) create mode 100644 man/air_dump_to_csv.Rd create mode 100644 man/air_generate_base_description.Rd create mode 100644 man/air_generate_metadata.Rd create mode 100644 man/air_get_base_description_from_table.Rd create mode 100644 man/air_get_metadata_from_table.Rd create mode 100644 man/flatten_col_to_chr.Rd diff --git a/DESCRIPTION b/DESCRIPTION index c39d8d3..19ed3d4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.1.6 +Version: 0.1.7 Date: 2017-09-17 Author: Darko Bergant Maintainer: Darko Bergant @@ -21,5 +21,6 @@ Imports: purrr, utils, snakecase, - tidyselect + tidyselect, + rlang RoxygenNote: 7.2.0 diff --git a/NAMESPACE b/NAMESPACE index 0bfc9db..42a7868 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,8 +3,13 @@ S3method(print,airtable.base) export(air_delete) export(air_dump) +export(air_dump_to_csv) +export(air_generate_base_description) +export(air_generate_metadata) export(air_get) export(air_get_attachments) +export(air_get_base_description_from_table) +export(air_get_metadata_from_table) export(air_get_schema) export(air_insert) export(air_insert_data_frame) @@ -15,6 +20,7 @@ export(air_update) export(air_update_data_frame) export(airtable) export(fetch_all) +export(flatten_col_to_chr) export(get_offset) export(get_unique_field_values) export(multiple) diff --git a/R/air_dump.R b/R/air_dump.R index 74c772f..263d820 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -1,131 +1,3 @@ -#' Output tables to a specified file format -#' -#' This function uses metadata tables in your base to extract all -#' base table to a different file format. Currently only CSV is supported. -#' For information about creating metadata tables in your base see the -#' \href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} -#' -#' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' -#' @param table_name_metadata String. Name of structural metadata table. PROVIDE LINK TO DEF OF STRUCTURAL METADATA -#' @param table_name_description String. Name of the descriptive metadata table. PROVIDE LINK TO DEF OF DESCRIPTIVE METADATA -#' @param output_dir String. Folder containing outputs. If not specified, a folder -#' with the name of the output_id will be used. -#' @param output_format String. Default is csv. Options include: csv -#' @param to_frictionless_data Logical. Requires table_name_metadata and table_name_description. -#' Should data be output as a frictionless data package. (To be Added) -#' @param all_table_names Vector. Do not use if providing table_name_metadata. -#' A character vector of all table names in your base. I.e c("table 1","table 2"). -#' @param output_id String. ID used to uniquely identify outputs. A folder with -#' this name will be created in output_dir. -#' -#' @return List of file paths created. -#' @export -#' -#' @examples - -# air_dump <- function(base = Sys.getenv("airtable_metadata_base"), -# table_name_metadata = "meta data", -# table_name_description = "description", -# all_table_names = NULL, -# output_dir = "outputs", -# output_id = rlang::hash(Sys.time()), -# output_format = "csv", -# to_frictionless_data=FALSE){ -# -# if(is.null(output_dir)){ -# output_dir <- "." -# message(glue::glue("No ouput_dir provided. -# Files are being saved directly to ./{output_id}")) -# } -# -# temp_path <- tempdir() -# ## create dir - should probably go to a temp dir until finished -# output_dir_path <- sprintf("%s/%s",temp_path,output_id) -# dir.create(path = output_dir_path,recursive = T, showWarnings = FALSE) -# -# ## check that output_format is supported -# output_format <- tolower(output_format) -# output_formats <- c("csv") # potentially include json, txt,xml -# if(!any(output_format %in% output_formats)){ -# stop(glue::glue("{output_format} not supported. Use one of {output_formats}")) -# } -# -# ## load metadata table - limit to just table names -# ### create structural metadata file -# -# # write structural metadata table -# str_metadata <- airtabler::fetch_all(base,table_name_metadata) -# smf_file_path <- sprintf("%s/%s.csv",output_dir_path,snakecase::to_snake_case(table_name_metadata)) -# # names(str_metadata) <- snakecase::to_lower_camel_case(names(str_metadata)) -# utils::write.csv(str_metadata,smf_file_path,row.names = FALSE) -# -# base_table_names <- unique(str_metadata$`Table Name`) -# base_table_names_formatted <- snakecase::to_snake_case(base_table_names) -# purrr::map2(base_table_names,base_table_names_formatted, function(x,y){ -# #browser() -# ## get fields from str_metadata -# -# fields_exp <- str_metadata[str_metadata$`Table Name` == x,"Field Name"] -# -# ## pull table - add check for blank tables -# x_table <- airtabler::fetch_all(base,x) -# -# if(!is.data.frame(x_table)){ -# x_table <- data.frame(id = character()) -# } -# -# ## add in missing columns -# fields_obs <- names(x_table) -# # drop autogenerated cols from comparison -# fields_obs <- fields_obs[!grepl(pattern = "^id$|^createdTime$", -# x = fields_obs)] -# # check if any discrepancy between metadata and table -# fields_diff <- set_diff(fields_exp,fields_obs) -# -# if(!is.null(fields_diff)){ -# # check for fields in obs not in exp - error -# obs_exp <- setdiff(fields_obs,fields_exp) -# if(length(obs_exp) != 0){ -# stop(glue::glue("Table {x} contains field(s) {obs_exp} not found in {table_name_metadata}. Update metadata table.")) -# } -# # check for fields in exp and not in obs - append unless frictionless -# if(!to_frictionless_data){ -# exp_obs <- setdiff(fields_exp,fields_obs) -# x_table[exp_obs] <- list(character(0)) -# } -# } -# -# ## clean up field names -# -# names(x_table) <- snakecase::to_snake_case(names(x_table)) -# -# # convert list type fields to strings -# -# x_table_flat <- dplyr::mutate(.data = x_table, -# dplyr::across(tidyselect:::where(is.list), purrr::flatten_chr)) -# -# ## export to CSV -# -# output_file_path <- sprintf("%s/%s.csv",output_dir_path,y) -# -# utils::write.csv(x_table_flat,output_file_path,row.names = FALSE) -# }) -# -# ## copy files from temp to final -# -# output_dir_path_final <- sprintf("%s/%s",output_dir,output_id) -# dir.create(output_dir_path_final,recursive = T) -# -# outputs_list <- list.files(output_dir_path,full.names = T) -# -# file.copy(from = outputs_list,to = output_dir_path_final,recursive = FALSE ,copy.mode = TRUE) -# -# message(glue::glue("Files can be found here: {output_dir_path_final}")) -# -# return(list.files(output_dir_path_final,full.names = T)) -# -# } - #' Get items that differ between x and y #' #' Unlike setdiff, this function creates the union of x and y then @@ -164,8 +36,20 @@ set_diff <- function(x,y){ return(diff) } -### get_metadata - returns a data frame - +#' Pull the metadata table from airtable +#' +#' For information about creating metadata tables in your base see the +#' \href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} +#' +#' @details Requires the following fields: table_name, field_name +#' +#' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' +#' @param table_name String. Name of structural metadata table - the metadata that +#' describes how tables and fields fit together. +#' +#' @return data.frame with metadata table +#' @export +#' air_get_metadata_from_table <- function(base, table_name){ # get structural metadata table str_metadata <- airtabler::fetch_all(base,table_name) @@ -183,7 +67,25 @@ air_get_metadata_from_table <- function(base, table_name){ return(str_metadata) } -### generate metadata - returns a data frame + +#' Generated Metadata from table names +#' +#' Generates a structural metadata table - the metadata that +#' describes how tables and fields fit together. Does not +#' include field types. +#' +#' @details For information about creating metadata tables in your base see the +#' \href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} +#' +#' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' +#' @param table_names Vector of strings. The names of your tables. eg c("table 1","table 2", etc.) +#' @param limit Number from 1-100. How many rows should we pull from each table to create the metdata? +#' Keep in mind that the airtable api will not return fields with "empty" values - "", false, or []. +#' Code runs faster if fewer rows are pulled. +#' +#' @return data.frame with structural metadata. +#' @export + air_generate_metadata <- function(base, table_names,limit=1){ warning('Airtable does not return fields with empty values - "", false, or []. It is better to create a specific metdata table and @@ -202,6 +104,19 @@ air_generate_metadata <- function(base, table_names,limit=1){ return(meta_data_table) } +#' Get base description from table +#' +#' Pull a table that has descriptive metadata. +#' Requires the following fields: +#' "title","primary_contact","email","base_description" +#' +#' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' +#' @param table_name String. Name of descriptive metadata table - the metadata that +#' describes the base and provides attribution +#' +#' @return data.frame with descriptive metadata. +#' @export + air_get_base_description_from_table<- function(base, table_name){ #fetch table desc_table <- airtabler::fetch_all(base,table_name) @@ -221,6 +136,40 @@ air_get_base_description_from_table<- function(base, table_name){ } +#' Generate descriptive metadata +#' +#' Creates a data.frame that describes the base. +#' +#' @details See \href{https://www.dublincore.org/resources/userguide/creating_metadata/}{dublin core} for inspiration about additional attributes. +#' +#' @param title String. Title is a property that refers to the name or names by +#' which a resource is formally known. +#' @param primary_contact String. Person or entity primarily responsible for +#' making the content of a resource +#' @param email String. Email of primary_contact +#' @param base_description String. This property refers to the description of +#' the content of a resource. The description is a potentially rich source of +#' indexable terms and assist the users in their selection of an appropriate +#' resource. +#' @param ... String. Additional descriptive metadata elements. See details. +#' Additional elements can be added as name pair values e.g. +#' \code{license = "CC BY 4.0", is_part_of = "https://doi.org/10.48321/MyDMP01"} +#' +#' @return data.frame with descriptive metadata +#' @export +#' +#' @examples +#' +#' air_generate_base_description(title = "My Awesome Base" , +#' primary_contact= "Base Creator/Maintainer", +#' email = "email@@example.com", +#' base_description = "This base is used to contain my awesome data +#' from a project studying XXX in YYY. Data in the base were collected +#' from 1900-01-01 to 1990-01-01 by researchers at Some Long Term Project.", +#' is_part_of = "https://doi.org/10.48321/MyDMP01", +#' is_part_of = "https://doi.org/10.5072/zenodo_sandbox.1062705" +#' ) +#' air_generate_base_description <- function(title = NA,primary_contact= NA,email = NA, base_description = NA,...){ desc_table <- data.frame(title,primary_contact,email,base_description,...) return(desc_table) @@ -228,6 +177,18 @@ air_generate_base_description <- function(title = NA,primary_contact= NA,email = ### extract_base - returns a named list +#' Dump all tables from a base into R +#' +#' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' +#' @param metadata Data.frame.Data frame with structural metadata - describes relationship between tables and fields. +#' @param description Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. +#' Can be left as NULL if base already contains a table called description. +#' @param add_missing_fields Logical. Should fields described in the metadata data.frame be added to corresponding tables? +#' +#' @return List of data.frames. All tables from metadata plus the +#' description and metadata tables. +#' @export +#' air_dump <- function(base, metadata, description = NULL, add_missing_fields = TRUE){ names(metadata) <- snakecase::to_snake_case(names(metadata)) @@ -284,21 +245,104 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR table_list$metadata <- metadata + # check for description table + named_description <- grepl(pattern = "description",x = names(table_list), ignore.case = TRUE) + + if(!is.null(description)){ + if(any(named_description)){ + warning("Base has a description table and a description data.frame was supplied to + this function. Inserting description data.frame at $description. Table + extract may be overwritten.") + } table_list$description <- description } else { - ## give null description - table_list$description <- air_generate_base_description() + ### description may already be in the base, think about + ## how best to handle this + if( + all( + !named_description + ) + ){ + ## give null description + table_list$description <- air_generate_base_description() + } } - table_list$description$date_created <- Sys.Date() + table_list[named_description][[1]]$created <- Sys.Date() return(table_list) } -### write to X +#' Flatten list columns to character +#' +#' Similar in spirit to purrr::flatten_chr except +#' that it can handle NULL values in lists and returns outputs +#' that can be written to csv. +#' +#' @details +#' Because the outputs are intended for use in CSV files, we must use +#' double quotes to indicate that the commas separating list values do +#' not delimit cells. This conforms to RFC 4180 standard for CSVs. +#' \url{https://datatracker.ietf.org/doc/html/rfc4180} +#' +#' @param data_frame a data frame, tibble or other data frame like object +#' +#' @return data_frame with list columns converted to character vectors. +#' @export +#' +#' @examples +#' +#' data_frame <- data.frame(a = I(list(list("Hello"), +#' list("Aloha"), +#' NULL, +#' list("Hola","Bonjour","Merhaba") +#' )), +#' b = 1:4, +#' c = letters[1:4] +#' ) +#' +#' test_df <- flatten_col_to_chr(data_frame) +#' +#' str(test_df) +#' +flatten_col_to_chr <- function(data_frame){ + for(i in names(data_frame)){ + # get column values + col_from_df <- data_frame[[i]] + + if(is.list(col_from_df)){ + ## create an object to hold character values + chr_col <- as.character() + for(j in 1:length(col_from_df)){ + list_element<- col_from_df[[j]] + if(is.null(list_element)){ + list_element <- "" + } + row_value<- sprintf('"%s"',paste(list_element,collapse = ",")) + + chr_col <- append(chr_col,row_value) + } + data_frame[i] <- chr_col + } + } + return(data_frame) +} +#' Save air_dump output to csv +#' +#' Saves data.frames from air_dump to csv files. File names are determined by +#' the names of the list objects from air_dump. Files will be saved in folder +#' with a unique name, inside the folder specified by \code{output_dir}. The +#' unique name is generated from a hash of the air_dump output. +#' +#' @param table_list List. List of data.frames output from \code{air_dump} +#' @param output_dir String. Folder containing output files +#' @param overwrite Logical. Should outputs be overwritten if they already exist? +#' +#' @return list. Returns the table_list object +#' @export air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE){ output_id <- rlang::hash(table_list) @@ -321,23 +365,8 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) ## clean up field names names(x_table) <- snakecase::to_snake_case(names(x_table)) - if(y_table_name == "table_2"){ - browser() - } - - purrr::map_dfc(x_table, function(x){ - if(is.list(x)){ - x <- purrr::flatten_dfc(x) - } - return(x) - }) - # convert list type fields to strings - x_table_flat <- dplyr::mutate(.data = x_table, - dplyr::across( - tidyselect:::where(is.list), - unlist) - ) + x_table_flat <- flatten_col_to_chr(x_table) ## export to CSV @@ -347,6 +376,19 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) }) } - +### write to db ### recover from metadata - JS code to regenerate tables +# +# air_js_for_tables <- function(metadata){ +# names(metadata) <- snakecase::to_snake_case(names(metadata)) +# table_names <- unique(metadata$table_name) +# +# ## not sufficient for all field types, need to think more deeply about this +# purrr::map(table_names, function(x){ +# field_names <- metadata[metadata$table_name == x,"field_name"] +# field_types <- metadata[metadata$table_name == x,"field_type"] +# create_field <- sprintf('{name: "%s", type: "%s"}',field_name,field_type) +# }) +# +# } diff --git a/man/air_dump.Rd b/man/air_dump.Rd index 5aeab14..4cfdc67 100644 --- a/man/air_dump.Rd +++ b/man/air_dump.Rd @@ -2,46 +2,24 @@ % Please edit documentation in R/air_dump.R \name{air_dump} \alias{air_dump} -\title{Output tables to a specified file format} +\title{Dump all tables from a base into R} \usage{ -air_dump( - base = Sys.getenv("airtable_metadata_base"), - table_name_metadata = "meta data", - table_name_description = "description", - all_table_names = NULL, - output_dir = "outputs", - output_id = rlang::hash(Sys.time()), - output_format = "csv", - to_frictionless_data = FALSE -) +air_dump(base, metadata, description = NULL, add_missing_fields = TRUE) } \arguments{ \item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} -\item{table_name_metadata}{String. Name of structural metadata table. PROVIDE LINK TO DEF OF STRUCTURAL METADATA} +\item{metadata}{Data.frame.Data frame with structural metadata - describes relationship between tables and fields.} -\item{table_name_description}{String. Name of the descriptive metadata table. PROVIDE LINK TO DEF OF DESCRIPTIVE METADATA} +\item{description}{Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. +Can be left as NULL if base already contains a table called description.} -\item{all_table_names}{Vector. Do not use if providing table_name_metadata. -A character vector of all table names in your base. I.e c("table 1","table 2").} - -\item{output_dir}{String. Folder containing outputs. If not specified, a folder -with the name of the output_id will be used.} - -\item{output_id}{String. ID used to uniquely identify outputs. A folder with -this name will be created in output_dir.} - -\item{output_format}{String. Default is csv. Options include: csv} - -\item{to_frictionless_data}{Logical. Requires table_name_metadata and table_name_description. -Should data be output as a frictionless data package. (To be Added)} +\item{add_missing_fields}{Logical. Should fields described in the metadata data.frame be added to corresponding tables?} } \value{ -List of file paths created. +List of data.frames. All tables from metadata plus the +description and metadata tables. } \description{ -This function uses metadata tables in your base to extract all -base table to a different file format. Currently only CSV is supported. -For information about creating metadata tables in your base see the -\href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} +Dump all tables from a base into R } diff --git a/man/air_dump_to_csv.Rd b/man/air_dump_to_csv.Rd new file mode 100644 index 0000000..1d59dc6 --- /dev/null +++ b/man/air_dump_to_csv.Rd @@ -0,0 +1,24 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_dump_to_csv} +\alias{air_dump_to_csv} +\title{Save air_dump output to csv} +\usage{ +air_dump_to_csv(table_list, output_dir = "outputs", overwrite = FALSE) +} +\arguments{ +\item{table_list}{List. List of data.frames output from \code{air_dump}} + +\item{output_dir}{String. Folder containing output files} + +\item{overwrite}{Logical. Should outputs be overwritten if they already exist?} +} +\value{ +list. Returns the table_list object +} +\description{ +Saves data.frames from air_dump to csv files. File names are determined by +the names of the list objects from air_dump. Files will be saved in folder +with a unique name, inside the folder specified by \code{output_dir}. The +unique name is generated from a hash of the air_dump output. +} diff --git a/man/air_generate_base_description.Rd b/man/air_generate_base_description.Rd new file mode 100644 index 0000000..3291517 --- /dev/null +++ b/man/air_generate_base_description.Rd @@ -0,0 +1,54 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_generate_base_description} +\alias{air_generate_base_description} +\title{Generate descriptive metadata} +\usage{ +air_generate_base_description( + title = NA, + primary_contact = NA, + email = NA, + base_description = NA, + ... +) +} +\arguments{ +\item{title}{String. Title is a property that refers to the name or names by +which a resource is formally known.} + +\item{primary_contact}{String. Person or entity primarily responsible for +making the content of a resource} + +\item{email}{String. Email of primary_contact} + +\item{base_description}{String. This property refers to the description of +the content of a resource. The description is a potentially rich source of +indexable terms and assist the users in their selection of an appropriate +resource.} + +\item{...}{String. Additional descriptive metadata elements. See details. +Additional elements can be added as name pair values e.g. +\code{license = "CC BY 4.0", is_part_of = "https://doi.org/10.48321/MyDMP01"}} +} +\value{ +data.frame with descriptive metadata +} +\description{ +Creates a data.frame that describes the base. +} +\details{ +See \href{https://www.dublincore.org/resources/userguide/creating_metadata/}{dublin core} for inspiration about additional attributes. +} +\examples{ + +air_generate_base_description(title = "My Awesome Base" , + primary_contact= "Base Creator/Maintainer", + email = "email@example.com", + base_description = "This base is used to contain my awesome data + from a project studying XXX in YYY. Data in the base were collected + from 1900-01-01 to 1990-01-01 by researchers at Some Long Term Project.", + is_part_of = "https://doi.org/10.48321/MyDMP01", + is_part_of = "https://doi.org/10.5072/zenodo_sandbox.1062705" + ) + +} diff --git a/man/air_generate_metadata.Rd b/man/air_generate_metadata.Rd new file mode 100644 index 0000000..b943357 --- /dev/null +++ b/man/air_generate_metadata.Rd @@ -0,0 +1,29 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_generate_metadata} +\alias{air_generate_metadata} +\title{Generated Metadata from table names} +\usage{ +air_generate_metadata(base, table_names, limit = 1) +} +\arguments{ +\item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} + +\item{table_names}{Vector of strings. The names of your tables. eg c("table 1","table 2", etc.)} + +\item{limit}{Number from 1-100. How many rows should we pull from each table to create the metdata? +Keep in mind that the airtable api will not return fields with "empty" values - "", false, or []. +Code runs faster if fewer rows are pulled.} +} +\value{ +data.frame with structural metadata. +} +\description{ +Generates a structural metadata table - the metadata that +describes how tables and fields fit together. Does not +include field types. +} +\details{ +For information about creating metadata tables in your base see the +\href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} +} diff --git a/man/air_get_base_description_from_table.Rd b/man/air_get_base_description_from_table.Rd new file mode 100644 index 0000000..a140403 --- /dev/null +++ b/man/air_get_base_description_from_table.Rd @@ -0,0 +1,22 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_get_base_description_from_table} +\alias{air_get_base_description_from_table} +\title{Get base description from table} +\usage{ +air_get_base_description_from_table(base, table_name) +} +\arguments{ +\item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} + +\item{table_name}{String. Name of descriptive metadata table - the metadata that +describes the base and provides attribution} +} +\value{ +data.frame with descriptive metadata. +} +\description{ +Pull a table that has descriptive metadata. +Requires the following fields: +"title","primary_contact","email","base_description" +} diff --git a/man/air_get_metadata_from_table.Rd b/man/air_get_metadata_from_table.Rd new file mode 100644 index 0000000..44a34dc --- /dev/null +++ b/man/air_get_metadata_from_table.Rd @@ -0,0 +1,24 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_get_metadata_from_table} +\alias{air_get_metadata_from_table} +\title{Pull the metadata table from airtable} +\usage{ +air_get_metadata_from_table(base, table_name) +} +\arguments{ +\item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} + +\item{table_name}{String. Name of structural metadata table - the metadata that +describes how tables and fields fit together.} +} +\value{ +data.frame with metadata table +} +\description{ +For information about creating metadata tables in your base see the +\href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} +} +\details{ +Requires the following fields: table_name, field_name +} diff --git a/man/air_get_schema.Rd b/man/air_get_schema.Rd index 61782d5..43039d3 100644 --- a/man/air_get_schema.Rd +++ b/man/air_get_schema.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/air_get_schema.R \name{air_get_schema} \alias{air_get_schema} -\title{Get base schema} +\title{Get base schema - enterprise only} \usage{ air_get_schema(base, ...) } @@ -15,6 +15,7 @@ air_get_schema(base, ...) list of schema } \description{ +Metadata API currently only available via enterprise accounts. Get the schema for the tables in a base. } \section{Using Metadata API}{ diff --git a/man/flatten_col_to_chr.Rd b/man/flatten_col_to_chr.Rd new file mode 100644 index 0000000..21f2988 --- /dev/null +++ b/man/flatten_col_to_chr.Rd @@ -0,0 +1,41 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{flatten_col_to_chr} +\alias{flatten_col_to_chr} +\title{Flatten list columns to character} +\usage{ +flatten_col_to_chr(data_frame) +} +\arguments{ +\item{data_frame}{a data frame, tibble or other data frame like object} +} +\value{ +data_frame with list columns converted to character vectors. +} +\description{ +Similar in spirit to purrr::flatten_chr except +that it can handle NULL values in lists and returns outputs +that can be written to csv. +} +\details{ +Because the outputs are intended for use in CSV files, we must use +double quotes to indicate that the commas separating list values do +not delimit cells. This conforms to RFC 4180 standard for CSVs. +\url{https://datatracker.ietf.org/doc/html/rfc4180} +} +\examples{ + +data_frame <- data.frame(a = I(list(list("Hello"), +list("Aloha"), +NULL, +list("Hola","Bonjour","Merhaba") +)), +b = 1:4, +c = letters[1:4] +) + +test_df <- flatten_col_to_chr(data_frame) + +str(test_df) + +} From cbcaf4730b852f68a75e446a9ec8e1a3b58e530d Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 30 Jun 2022 16:15:47 -0500 Subject: [PATCH 035/126] updated missing field warnings --- R/air_dump.R | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index 263d820..105c02a 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -192,6 +192,7 @@ air_generate_base_description <- function(title = NA,primary_contact= NA,email = air_dump <- function(base, metadata, description = NULL, add_missing_fields = TRUE){ names(metadata) <- snakecase::to_snake_case(names(metadata)) + ## check for required fields required_fields <- c("table_name","field_name") @@ -204,11 +205,10 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR base_table_names <- unique(metadata$table_name) - table_list <- base_table_names |> purrr::set_names() |> purrr::map(function(x){ - #browser() + ## get fields from str_metadata fields_exp <- metadata[metadata$table_name == x,"field_name"] @@ -225,12 +225,18 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR # check if any discrepancy between metadata and table fields_diff <- set_diff(fields_exp,fields_obs) - + #browser() if(!is.null(fields_diff)){ # check for fields in obs not in exp - error obs_exp <- setdiff(fields_obs,fields_exp) - if(length(obs_exp) != 0 & !all(obs_exp %in% c("id","createdTime"))){ - stop(glue::glue("Table {x} contains field(s) {obs_exp} not found in {table_name_metadata}. Update metadata table.")) + ignore_fields <- c("id","createdTime") + ignore_fields_pattern <- paste(ignore_fields,collapse = "|") + if(length(obs_exp) != 0 & !all(obs_exp %in% ignore_fields)){ + missing_fields <- obs_exp[!grepl(ignore_fields_pattern,obs_exp,ignore.case = FALSE)] + missing_fields_glue <- paste(missing_fields, collapse = ", ") + stop(glue::glue('The metadata table is missing the following fields from table {x}: + {missing_fields_glue} + Please update the metadata table.https://airtable.com/{base}')) } # check for fields in exp and not in obs - append unless frictionless if(add_missing_fields){ From 27ce6db2583c0f106fb317423286b454d20d1993 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 30 Jun 2022 17:05:32 -0500 Subject: [PATCH 036/126] Update issue templates --- .github/ISSUE_TEMPLATE/blank-issue.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/blank-issue.md diff --git a/.github/ISSUE_TEMPLATE/blank-issue.md b/.github/ISSUE_TEMPLATE/blank-issue.md new file mode 100644 index 0000000..a2a56c3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/blank-issue.md @@ -0,0 +1,10 @@ +--- +name: Blank Issue +about: Use this issue template for issues that are not bugs or enhancements +title: '' +labels: '' +assignees: '' + +--- + + From 5fdc06be7ea5ebe899cd2696fe9132cc5a3ea677 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 30 Jun 2022 17:10:12 -0500 Subject: [PATCH 037/126] Update issue templates --- .github/ISSUE_TEMPLATE/bug_report.md | 27 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 17 ++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..912f517 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,27 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Code to reproduce the behavior: + +**Expected behavior** +A clear and concise description of what you expected to happen. + + +**Session info (please complete the following information):** + +``` +sessionInfo() +``` + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..823ca61 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +**Describe why this feature should be added** +A clear and concise description of what the need is. + +**Describe the behavior you expect** +A clear and concise description of what you want to happen. + +**Additional context** +Add any other context or screenshots about the feature request here. From 73771239db73ab5d33c76bfd707b3ea64d48bd01 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 30 Jun 2022 17:23:23 -0500 Subject: [PATCH 038/126] added data frame handling --- R/air_dump.R | 48 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index 105c02a..276c649 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -205,10 +205,11 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR base_table_names <- unique(metadata$table_name) + + print(base_table_names) table_list <- base_table_names |> purrr::set_names() |> purrr::map(function(x){ - ## get fields from str_metadata fields_exp <- metadata[metadata$table_name == x,"field_name"] @@ -251,6 +252,7 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR table_list$metadata <- metadata + #browser() # check for description table named_description <- grepl(pattern = "description",x = names(table_list), ignore.case = TRUE) @@ -275,7 +277,9 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR } } - table_list[named_description][[1]]$created <- Sys.Date() + named_description_post <- grepl(pattern = "description",x = names(table_list), ignore.case = TRUE) + + table_list[named_description_post][[1]]$created <- Sys.Date() return(table_list) } @@ -306,7 +310,8 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR #' list("Hola","Bonjour","Merhaba") #' )), #' b = 1:4, -#' c = letters[1:4] +#' c = letters[1:4], +#' d = I(data.frame(id = 132, name = "bob", email = "bob@@example.com")) #' ) #' #' test_df <- flatten_col_to_chr(data_frame) @@ -315,21 +320,41 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR #' flatten_col_to_chr <- function(data_frame){ for(i in names(data_frame)){ + browser() # get column values col_from_df <- data_frame[[i]] if(is.list(col_from_df)){ ## create an object to hold character values chr_col <- as.character() - for(j in 1:length(col_from_df)){ - list_element<- col_from_df[[j]] - if(is.null(list_element)){ - list_element <- "" + if(is.data.frame(col_from_df)){ + n_r <- nrow(col_from_df) + + for(j in 1:n_r){ + list_element<- col_from_df[j,] + if(is.null(list_element)){ + list_element <- "" + } + row_value<- sprintf('"%s"',paste(list_element,collapse = ",")) + + chr_col <- append(chr_col,row_value) + } + + } else { + n <- length(col_from_df) + + for(j in 1:n){ + list_element<- col_from_df[[j]] + if(is.null(list_element)){ + list_element <- "" + } + row_value<- sprintf('"%s"',paste(list_element,collapse = ",")) + + chr_col <- append(chr_col,row_value) } - row_value<- sprintf('"%s"',paste(list_element,collapse = ",")) - chr_col <- append(chr_col,row_value) } + data_frame[i] <- chr_col } } @@ -356,9 +381,11 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) # check if data already exist if(dir.exists(output_dir_path) & !overwrite){ - message("data already exist, files not written") + message("data already exist, files not written. Set overwrite + to TRUE ") return(list.files(output_dir_path)) } + ### consider using temp dir then copying once finished processing dir.create(output_dir_path,recursive = TRUE) @@ -368,7 +395,6 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) ## clean up field names in table names(x_table) <- snakecase::to_snake_case(names(x_table)) - ## clean up field names names(x_table) <- snakecase::to_snake_case(names(x_table)) From 54cc1fdce4246bf82fee8c432015f0e1644c0469 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 30 Jun 2022 17:26:32 -0500 Subject: [PATCH 039/126] dropped browser from air_dump --- R/air_dump.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/air_dump.R b/R/air_dump.R index 276c649..bbc46cc 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -320,7 +320,7 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR #' flatten_col_to_chr <- function(data_frame){ for(i in names(data_frame)){ - browser() + #browser() # get column values col_from_df <- data_frame[[i]] @@ -328,6 +328,7 @@ flatten_col_to_chr <- function(data_frame){ ## create an object to hold character values chr_col <- as.character() if(is.data.frame(col_from_df)){ + n_r <- nrow(col_from_df) for(j in 1:n_r){ From 64ffe867074e4c864f9d341cf4c61b930d0fbfcb Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 30 Jun 2022 18:13:24 -0500 Subject: [PATCH 040/126] csv writes to temp first and fixed example in flatten --- R/air_dump.R | 25 ++++++++++++++++++++----- man/flatten_col_to_chr.Rd | 3 ++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index bbc46cc..a536606 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -311,7 +311,7 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR #' )), #' b = 1:4, #' c = letters[1:4], -#' d = I(data.frame(id = 132, name = "bob", email = "bob@@example.com")) +#' d = I(data.frame(id = 1:4, name = "bob", email = "bob@@example.com")) #' ) #' #' test_df <- flatten_col_to_chr(data_frame) @@ -377,15 +377,21 @@ flatten_col_to_chr <- function(data_frame){ #' @export air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE){ + # create a unique id for the data output_id <- rlang::hash(table_list) - output_dir_path <- sprintf("%s/%s",output_dir,output_id) - # check if data already exist - if(dir.exists(output_dir_path) & !overwrite){ + # check if data already exist + output_dir_path_final <- sprintf("%s/%s",output_dir,output_id) + if(dir.exists(output_dir_path_final) & !overwrite){ message("data already exist, files not written. Set overwrite to TRUE ") - return(list.files(output_dir_path)) + return(list.files(output_dir_path_final,full.names = TRUE)) } + + # create temp dir + temp_path <- tempdir() + output_dir_path <- sprintf("%s/%s",temp_path,output_id) + ### consider using temp dir then copying once finished processing dir.create(output_dir_path,recursive = TRUE) @@ -407,6 +413,15 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) utils::write.csv(x_table_flat,output_file_path,row.names = FALSE) }) + + ## copy from temp to final + dir.create(output_dir_path_final,recursive = TRUE) + outputs_list <- list.files(output_dir_path,full.names = T) + + file.copy(from = outputs_list,to = output_dir_path_final,recursive = FALSE ,copy.mode = TRUE) + + invisible(table_list) + } ### write to db diff --git a/man/flatten_col_to_chr.Rd b/man/flatten_col_to_chr.Rd index 21f2988..7d2dbf6 100644 --- a/man/flatten_col_to_chr.Rd +++ b/man/flatten_col_to_chr.Rd @@ -31,7 +31,8 @@ NULL, list("Hola","Bonjour","Merhaba") )), b = 1:4, -c = letters[1:4] +c = letters[1:4], +d = I(data.frame(id = 1:4, name = "bob", email = "bob@example.com")) ) test_df <- flatten_col_to_chr(data_frame) From 9f5e3a277ce44b531f3bf52b041c7136a4a9d492 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 11 Aug 2022 12:06:50 -0500 Subject: [PATCH 041/126] udpated readme with instructions for editing renviron --- readme.md | 98 ++++++++++-------------------------------------------- readme.rmd | 17 +++++++--- 2 files changed, 31 insertions(+), 84 deletions(-) diff --git a/readme.md b/readme.md index 301d081..2fe56ec 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,9 @@ -# airtabler +--- +title: "airtabler" +output: + html_document: + keep_md: yes +--- Provides access to the [Airtable API](http://airtable.com/api) @@ -20,10 +25,19 @@ Generate the airtable API key from your [Airtable account](http://airtable.com/a __airtabler__ functions will read the API key from environment variable `AIRTABLE_API_KEY`. To start R session with the - initialized environvent variable create an `.Renviron` file in your home directory - with a line like this: + initialized environvent variable create an `.Renviron` file in your home directory. -`AIRTABLE_API_KEY=your_api_key_here` +```r +usethis::edit_r_environ +``` + +In .Renviron add the following: +``` +AIRTABLE_API_KEY=your_api_key_here + +``` + +**NOTE: Be sure the last line of your .Renviron file is an empty return line** To check where your home is, type `path.expand("~")` in your R console. @@ -55,18 +69,6 @@ hotels <- knitr::kable(hotels[, c("id","Name", "Stars", "Price/night")], format = "markdown") ``` - - -|id |Name |Stars | Price/night| -|:-----------------|:------------------------------------------------------------|:-----|-----------:| -|reccPOcMQaYt1tthb |Heritage Christchurch Hotel (Christchurch, New Zealand) |**** | 176| -|receHGZJ22WyUxocl |Urikana Boutique Hotel (Teresopolis, Brazil) |***** | 146| -|recgKO7K15YyWEsdb |Radisson Blu Hotel Marseille Vieux Port (Marseilles, France) |**** | 170| -|recjJJ4TX38sUwzfj |Hotel Berg (Keflavík, Iceland) |*** | 136| -|recjUU2GT28yVvw7l |Sheraton Nha Trang (Nha Trang, Vietnam) |***** | 136| -|reckPH6G384y3suac |Grand Residences Riviera Cancun (Puerto Morelos, Mexico) |***** | 278| -|reclG7Bd2g5Dtiw4J |Grand Budapest Hotel (Zubrowka) |***** | 156| - Filter records with formula (see [formula field reference ](https://support.airtable.com/hc/en-us/articles/203255215-Formula-Field-Reference)). @@ -77,17 +79,6 @@ hotels <- knitr::kable(hotels[, c("id","Name", "Stars", "Avg Review", "Price/night")], format = "markdown") ``` - - -|id |Name |Stars | Avg Review| Price/night| -|:-----------------|:--------------------------------------------------------|:-----|----------:|-----------:| -|reccPOcMQaYt1tthb |Heritage Christchurch Hotel (Christchurch, New Zealand) |**** | 8.8| 176| -|receHGZJ22WyUxocl |Urikana Boutique Hotel (Teresopolis, Brazil) |***** | 9.0| 146| -|recjJJ4TX38sUwzfj |Hotel Berg (Keflavík, Iceland) |*** | 9.2| 136| -|recjUU2GT28yVvw7l |Sheraton Nha Trang (Nha Trang, Vietnam) |***** | 8.8| 136| -|reckPH6G384y3suac |Grand Residences Riviera Cancun (Puerto Morelos, Mexico) |***** | 9.1| 278| -|reclG7Bd2g5Dtiw4J |Grand Budapest Hotel (Zubrowka) |***** | 9.0| 156| - Sort data with sort parameter: ```r @@ -101,18 +92,6 @@ hotels <- knitr::kable(hotels[, c("id","Name", "Stars", "Avg Review", "Price/night")], format = "markdown") ``` - - -|id |Name |Stars | Avg Review| Price/night| -|:-----------------|:------------------------------------------------------------|:-----|----------:|-----------:| -|recjJJ4TX38sUwzfj |Hotel Berg (Keflavík, Iceland) |*** | 9.2| 136| -|reckPH6G384y3suac |Grand Residences Riviera Cancun (Puerto Morelos, Mexico) |***** | 9.1| 278| -|receHGZJ22WyUxocl |Urikana Boutique Hotel (Teresopolis, Brazil) |***** | 9.0| 146| -|reclG7Bd2g5Dtiw4J |Grand Budapest Hotel (Zubrowka) |***** | 9.0| 156| -|recjUU2GT28yVvw7l |Sheraton Nha Trang (Nha Trang, Vietnam) |***** | 8.8| 136| -|reccPOcMQaYt1tthb |Heritage Christchurch Hotel (Christchurch, New Zealand) |**** | 8.8| 176| -|recgKO7K15YyWEsdb |Radisson Blu Hotel Marseille Vieux Port (Marseilles, France) |**** | 8.2| 170| - ### Using page size and offset Define page size with `pageSize`: @@ -122,10 +101,6 @@ hotels <- TravelBucketList$Hotels$select(pageSize = 3) nrow(hotels) ``` -``` -## [1] 3 -``` - Continue at offset, returned by previous select: ```r @@ -133,10 +108,6 @@ hotels <- TravelBucketList$Hotels$select(offset = get_offset(hotels)) nrow(hotels) ``` -``` -## [1] 4 -``` - To fetch all rows (even > 100 records) use `select_all`. The `select_all` function will handle the offset and return the result as a single object. @@ -147,10 +118,6 @@ hotels <- TravelBucketList$Hotels$select_all() nrow(hotels) ``` -``` -## [1] 7 -``` - Other optional arguments: @@ -170,19 +137,6 @@ radisson <- str(radisson$fields, max.level = 1) ``` -``` -## List of 9 -## $ Listing URL: chr "https://www.booking.com/hotel/fr/radisson-sas-marseille-vieux-port.html" -## $ Name : chr "Radisson Blu Hotel Marseille Vieux Port (Marseilles, France)" -## $ Price/night: int 170 -## $ Amenities : chr [1:4] "Pool" "Gym" "Restaurant" "Wifi" -## $ Notes : chr "Rooms with African or Provencál decor." -## $ Country : chr "recmSV4PR9ZCWyrk8" -## $ Pictures :'data.frame': 4 obs. of 6 variables: -## $ Stars : chr "****" -## $ Avg Review : num 8.2 -``` - ### Insert a record Insert a new record with `insert` function (API returns all record data - including new record ID): @@ -201,10 +155,6 @@ new_hotel <- cat("Inserted a record with ID=", new_hotel$id, sep = "") ``` -``` -## Inserted a record with ID=recGtWMprUr7f2EvT -``` - ### Update a record Update the price of the new hotel (API returns all record data): @@ -223,24 +173,12 @@ cat("Updated a record with ID=", new_hotel$id, ". ", "New price: ", new_hotel$fields$`Price/night`, sep = "") ``` -``` -## Updated a record with ID=recGtWMprUr7f2EvT. New price: 120 -``` - ### Delete a record ```r TravelBucketList$Hotels$delete(new_hotel$id) ``` -``` -## $deleted -## [1] TRUE -## -## $id -## [1] "recGtWMprUr7f2EvT" -``` - ## Working with data frames diff --git a/readme.rmd b/readme.rmd index 4d6a6d3..f5d7f7d 100644 --- a/readme.rmd +++ b/readme.rmd @@ -7,7 +7,7 @@ output: Provides access to the [Airtable API](http://airtable.com/api) ```{r setup, include=FALSE} -knitr::opts_chunk$set(echo = TRUE) +knitr::opts_chunk$set(echo = TRUE, eval = FALSE) ``` ## Install @@ -26,10 +26,19 @@ Generate the airtable API key from your [Airtable account](http://airtable.com/a __airtabler__ functions will read the API key from environment variable `AIRTABLE_API_KEY`. To start R session with the - initialized environvent variable create an `.Renviron` file in your home directory - with a line like this: + initialized environvent variable create an `.Renviron` file in your home directory. -`AIRTABLE_API_KEY=your_api_key_here` +```r +usethis::edit_r_environ +``` + +In .Renviron add the following: +``` +AIRTABLE_API_KEY=your_api_key_here + +``` + +**NOTE: Be sure the last line of your .Renviron file is an empty return line** To check where your home is, type `path.expand("~")` in your R console. From 5c0f3fd6728de3fabdaeee037ea92199fc4b7026 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 19 Aug 2022 17:45:25 -0400 Subject: [PATCH 042/126] functions to get data as raw json --- R/air_get_json.R | 93 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 R/air_get_json.R diff --git a/R/air_get_json.R b/R/air_get_json.R new file mode 100644 index 0000000..f679657 --- /dev/null +++ b/R/air_get_json.R @@ -0,0 +1,93 @@ +#' Get a list of records or retrieve a single record as JSON +#' +#' Returns JSON objects from GET requests +#' +#' @param base Airtable base +#' @param table_name Table name +#' @param record_id (optional) Use record ID argument to retrieve an existing +#' record details +#' @param limit (optional) A limit on the number of records to be returned. +#' Limit can range between 1 and 100. +#' @param offset (optional) Page offset returned by the previous list-records +#' call. Note that this is represented by a record ID, not a numerical offset. +#' @param view (optional) The name or ID of the view +#' @param sortField (optional) The field name to use for sorting +#' @param sortDirection (optional) "asc" or "desc". The sort order in which the +#' records will be returned. Defaults to asc. +#' @param combined_result If TRUE (default) all data is returned in the same data. +#' If FALSE table fields are returned in separate \code{fields} element. +#' @param fields (optional) Only data for fields whose names are in this list +#' will be included in the records. If you don't need every field, you can use +#' @param pretty Logical. Should JSON be returned in human readable form? +#' this parameter to reduce the amount of data transferred. +#' @return A data frame with records or a list with record details if +#' \code{record_id} is specified. +#' @export +air_get_json <- function(base, table_name, + record_id = NULL, + limit = NULL, + offset = NULL, + view = NULL, + fields = NULL, + sortField = NULL, + sortDirection = NULL, + combined_result = TRUE, + pretty = FALSE) { + + search_path <- table_name + + if(!missing(record_id)) { + search_path <- paste0(search_path, "/", record_id) + } + request_url <- sprintf("%s/%s/%s?", air_url, base, search_path) + request_url <- utils::URLencode(request_url) + + # append parameters to URL: + param_list <- as.list(environment())[c( + "limit", "offset", "view", "sortField", "sortDirection")] + param_list <- param_list[!sapply(param_list, is.null)] + if(!is.null(fields)) { + param_list <- c(param_list, list_params(x = fields, par_name = "fields")) + } + + request_url <- httr::modify_url(request_url, query = param_list) + request_url <- gsub(pattern = "fields=",replacement = "fields%5B%5D=",x = request_url) + + # call service: + res <- httr::GET( + url = request_url, + config = httr::add_headers(Authorization = paste("Bearer", air_api_key())) + ) + air_validate(res) # throws exception (stop) if error + ret <- httr::content(res, as = "text") #returns json object + if(pretty){ + ret <- jsonlite::prettify(ret) + } + return(ret) + +} + + +fetch_all_json <- function(base, table_name, ...) { + out <- list() + out[[1]] <- airtabler::air_get_json(base, table_name, combined_result = FALSE,...) + if(length(out[[1]]) == 0){ + emptyTableMessage <- glue::glue("The queried view for {table_name} in {base} is empty") + warning(emptyTableMessage) + return(emptyTableMessage) + } else { + offset <- airtabler::get_offset(airtabler::air_parse(out[[1]])) #parse out offset + while (!is.null(offset)) { + out <- c(out, list(airtabler::air_get_json(base, table_name, combined_result = FALSE, offset = offset, ...))) + offset <- airtabler::get_offset(airtabler::air_parse(out[[length(out)]])) + } + + ## now we have a list of json text, paste together + # remove offset attributes and object delimiters {} + # from + + out <- dplyr::bind_rows(out) + cbind(id = out$id, out$fields, createdTime = out$createdTime, + stringsAsFactors = FALSE) + } +} From 8f3ab07ad5aa5d23e5b4e84bc4ab287037ee3005 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 22 Aug 2022 10:26:47 -0400 Subject: [PATCH 043/126] changed column order so that field is primary key --- R/air_dump.R | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) diff --git a/R/air_dump.R b/R/air_dump.R index a536606..b2e5ea4 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -96,7 +96,7 @@ air_generate_metadata <- function(base, table_names,limit=1){ ## guess record types? - md_df <- data.frame(table_name = x, field_name = fields_x, field_desc = "", field_type = "") + md_df <- data.frame( field_name = fields_x, table_name = x, field_desc = "", field_type = "") return(md_df) }) @@ -297,6 +297,7 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR #' not delimit cells. This conforms to RFC 4180 standard for CSVs. #' \url{https://datatracker.ietf.org/doc/html/rfc4180} #' +#' #' @param data_frame a data frame, tibble or other data frame like object #' #' @return data_frame with list columns converted to character vectors. @@ -362,6 +363,8 @@ flatten_col_to_chr <- function(data_frame){ return(data_frame) } + + #' Save air_dump output to csv #' #' Saves data.frames from air_dump to csv files. File names are determined by @@ -424,6 +427,88 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) } + +### Get just the json to preserve the structure -- essentially modified air_get + + + +### extract_base - returns a named list + +#' Dump all tables from a base into json files +#' +#' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' +#' @param metadata Data.frame.Data frame with structural metadata - describes relationship between tables and fields. +#' @param description Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. +#' Can be left as NULL if base already contains a table called description. +#' +#' @return List of data.frames. All tables from metadata plus the +#' description and metadata tables. +#' @export +#' +air_dump_to_json <- function(base, metadata, description = NULL, add_missing_fields = TRUE){ +# +# names(metadata) <- snakecase::to_snake_case(names(metadata)) +# +# ## check for required fields +# required_fields <- c("table_name","field_name") +# +# if(!all(required_fields %in% names(metadata))){ +# stop(glue::glue("metadata table must contain the +# following fields: {required_fields}. Note +# that field names are converted to snakecase +# before check.")) +# } +# +# +# base_table_names <- unique(metadata$table_name) +# +# print(base_table_names) +# table_list <- base_table_names |> +# purrr::set_names() |> +# purrr::map(function(x){ +# ## get fields from str_metadata +# +# fields_exp <- metadata[metadata$table_name == x,"field_name"] +# +# ## pull table - add check for blank tables +# x_table <- airtabler::fetch_all(base,x) +# +# if(!is.data.frame(x_table)){ +# x_table <- data.frame(id = character()) +# } +# +# ## add in missing columns if any +# fields_obs <- names(x_table) +# +# # check if any discrepancy between metadata and table +# fields_diff <- set_diff(fields_exp,fields_obs) +# #browser() +# if(!is.null(fields_diff)){ +# # check for fields in obs not in exp - error +# obs_exp <- setdiff(fields_obs,fields_exp) +# ignore_fields <- c("id","createdTime") +# ignore_fields_pattern <- paste(ignore_fields,collapse = "|") +# if(length(obs_exp) != 0 & !all(obs_exp %in% ignore_fields)){ +# missing_fields <- obs_exp[!grepl(ignore_fields_pattern,obs_exp,ignore.case = FALSE)] +# missing_fields_glue <- paste(missing_fields, collapse = ", ") +# stop(glue::glue('The metadata table is missing the following fields from table {x}: +# {missing_fields_glue} +# Please update the metadata table.https://airtable.com/{base}')) +# } +# # check for fields in exp and not in obs - append unless frictionless +# if(add_missing_fields){ +# exp_obs <- setdiff(fields_exp,fields_obs) +# x_table[exp_obs] <- list(character(0)) +# } +# } +# +# return(x_table) +# +# }) +} + + + ### write to db ### recover from metadata - JS code to regenerate tables From 2fd2a5468f5bddbf6398585ffdf1736d191b8d1d Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 23 Aug 2022 14:59:13 -0400 Subject: [PATCH 044/126] added option to parse text in air_parse --- R/air_get_json.R | 24 +++++++++++++++++++----- R/airtabler.R | 6 +++++- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/R/air_get_json.R b/R/air_get_json.R index f679657..81deb8a 100644 --- a/R/air_get_json.R +++ b/R/air_get_json.R @@ -70,21 +70,35 @@ air_get_json <- function(base, table_name, fetch_all_json <- function(base, table_name, ...) { out <- list() - out[[1]] <- airtabler::air_get_json(base, table_name, combined_result = FALSE,...) + out[[1]] <- air_get_json(base, table_name, combined_result = FALSE,...) if(length(out[[1]]) == 0){ emptyTableMessage <- glue::glue("The queried view for {table_name} in {base} is empty") warning(emptyTableMessage) return(emptyTableMessage) } else { - offset <- airtabler::get_offset(airtabler::air_parse(out[[1]])) #parse out offset + offset <- airtabler::get_offset(air_parse(out[[1]])) #parse out offset while (!is.null(offset)) { - out <- c(out, list(airtabler::air_get_json(base, table_name, combined_result = FALSE, offset = offset, ...))) - offset <- airtabler::get_offset(airtabler::air_parse(out[[length(out)]])) + json_text <- list(air_get_json(base, table_name, combined_result = FALSE, offset = offset, ...)) + out <- c(out, json_text) + offset <- airtabler::get_offset(air_parse(out[[length(out)]])) } ## now we have a list of json text, paste together # remove offset attributes and object delimiters {} - # from + + # flatten list + out_flat <- purrr::flatten(out) + # paste collapse with delimiter + json_delimited <- paste(out_flat,collapse = "##########") + # remove delimiter and unnecessary json + # gsub behind ahead + json_open_preceeding_obj <- gsub(pattern = "\\],\"offset\":\"[:alnum:]{17,}/[:alnum:]{17,}\"}(?<##########)", + replacement = ",", + x = json_delimited) + # gsub look head + json_open_following_obj <- gsub(pattern = "(?<=##########)],{\"records\"\\:\\[", + replacement = "", + x = json_delimited) out <- dplyr::bind_rows(out) cbind(id = out$id, out$fields, createdTime = out$createdTime, diff --git a/R/airtabler.R b/R/airtabler.R index c4e1725..1d6168a 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -276,7 +276,11 @@ air_validate <- function(res) { } air_parse <- function(res) { - res_obj <- jsonlite::fromJSON(httr::content(res, as = "text")) + if(is.character(res)){ + res_obj<- jsonlite::fromJSON(res) + } else { + res_obj <- jsonlite::fromJSON(httr::content(res, as = "text")) + } if(!is.null(res_obj$records)) { res <- res_obj$records if(!is.null(res_obj$offset)) { From 45aa6ba9886bec83f2634923b2e628433757230c Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 23 Aug 2022 15:12:16 -0400 Subject: [PATCH 045/126] skip lines in excel reads --- DESCRIPTION | 2 +- R/air_get_attachments.R | 5 +++-- man/air_get_attachments.Rd | 3 +++ man/air_update_data_frame.Rd | 3 --- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index f11ddfd..4be95bb 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -20,4 +20,4 @@ Imports: curl, purrr, utils -RoxygenNote: 7.1.1 +RoxygenNote: 7.2.1 diff --git a/R/air_get_attachments.R b/R/air_get_attachments.R index a5f8cde..1e91186 100644 --- a/R/air_get_attachments.R +++ b/R/air_get_attachments.R @@ -12,12 +12,13 @@ #' @param download_file Logical. Should files be downloaded? #' @param dir_name String. Where should files be downloaded to? #' Will create the folder if it does not exist. +#' @param skip Numeric. How many lines should be skipped? See \code{read_excel} skip. #' #' @return named list of data frames #' @export air_get_attachments #' #' @examples -air_get_attachments <- function(base, table_name, field, download_file = FALSE, dir_name = "downloads", extract_type ="excel", extract_field ="excel_extract", ...){ +air_get_attachments <- function(base, table_name, field, download_file = FALSE, dir_name = "downloads", extract_type ="excel", extract_field ="excel_extract", skip = 0, ...){ #browser() # get data x <- fetch_all(base,table_name,...) @@ -59,7 +60,7 @@ air_get_attachments <- function(base, table_name, field, download_file = FALSE, warning(sprintf("Record ID %s is null",ID)) return(NULL) } - read_excel_url(x$url) ## need to be able to pass additional arguments + read_excel_url(x$url, skip = skip) ## need to be able to pass additional arguments }) ## add extract to data frame ---- diff --git a/man/air_get_attachments.Rd b/man/air_get_attachments.Rd index cf9355e..54ca157 100644 --- a/man/air_get_attachments.Rd +++ b/man/air_get_attachments.Rd @@ -12,6 +12,7 @@ air_get_attachments( dir_name = "downloads", extract_type = "excel", extract_field = "excel_extract", + skip = 0, ... ) } @@ -32,6 +33,8 @@ Should be one of: excel} \item{extract_field}{String. Name of extract field that will be created} +\item{skip}{Numeric. How many lines should be skipped? See \code{read_excel} skip.} + \item{...}{Additional arguments to pass to \code{air_get}} } \value{ diff --git a/man/air_update_data_frame.Rd b/man/air_update_data_frame.Rd index db23c37..375cd71 100644 --- a/man/air_update_data_frame.Rd +++ b/man/air_update_data_frame.Rd @@ -14,9 +14,6 @@ air_update_data_frame(base, table_name, record_ids, records) \item{record_ids}{Vector of strings. Records to be modified} \item{records}{Dataframe. Values to update} -} -\value{ - } \description{ Updates the values in a table by overwriting their current contents. From 3ceef2f2b313a002192c4c164fe1f1edcf48c73c Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 23 Aug 2022 15:14:19 -0400 Subject: [PATCH 046/126] updated docs so function reference is more specific --- R/air_get_attachments.R | 2 +- man/air_get_attachments.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/air_get_attachments.R b/R/air_get_attachments.R index 1e91186..f2bf773 100644 --- a/R/air_get_attachments.R +++ b/R/air_get_attachments.R @@ -12,7 +12,7 @@ #' @param download_file Logical. Should files be downloaded? #' @param dir_name String. Where should files be downloaded to? #' Will create the folder if it does not exist. -#' @param skip Numeric. How many lines should be skipped? See \code{read_excel} skip. +#' @param skip Numeric. How many lines should be skipped? See \code{readxl::read_excel} skip. #' #' @return named list of data frames #' @export air_get_attachments diff --git a/man/air_get_attachments.Rd b/man/air_get_attachments.Rd index 54ca157..fba4398 100644 --- a/man/air_get_attachments.Rd +++ b/man/air_get_attachments.Rd @@ -33,7 +33,7 @@ Should be one of: excel} \item{extract_field}{String. Name of extract field that will be created} -\item{skip}{Numeric. How many lines should be skipped? See \code{read_excel} skip.} +\item{skip}{Numeric. How many lines should be skipped? See \code{readxl::read_excel} skip.} \item{...}{Additional arguments to pass to \code{air_get}} } From 8ac50b62c2a4eb5c3505942546c6aae9d4731a66 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 23 Aug 2022 15:25:37 -0400 Subject: [PATCH 047/126] updated description file for our version of the package --- DESCRIPTION | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 4be95bb..9a69d2d 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,15 +1,15 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.1.6 -Date: 2017-09-17 +Version: 0.1.7 +Date: 2022-08-23 Author: Darko Bergant -Maintainer: Darko Bergant -Description: Provides access to the Airtable (airtable.com) API. +Maintainer: Collin Schwantes +Description: Fork from Darko Bergant's package. Provides access to the Airtable (airtable.com) API. Depends: R (>= 3.2.0) License: MIT + file LICENSE -URL: https://github.com/bergant/airtabler +URL: https://github.com/ecohealthalliance/airtabler LazyData: TRUE Imports: glue, From a3d16ca0dcf9a97aeb75a360539d820710f051cc Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Wed, 24 Aug 2022 12:19:22 -0400 Subject: [PATCH 048/126] added filterByFormula paramter to air_get --- DESCRIPTION | 2 +- R/airtabler.R | 5 ++++- man/air_get.Rd | 7 +++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 9a69d2d..928f62b 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.1.7 +Version: 0.1.8 Date: 2022-08-23 Author: Darko Bergant Maintainer: Collin Schwantes diff --git a/R/airtabler.R b/R/airtabler.R index c4e1725..4c699dd 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -70,6 +70,7 @@ air_secret_key <- function(){ #' If FALSE table fields are returned in separate \code{fields} element. #' @param fields (optional) Only data for fields whose names are in this list #' will be included in the records. If you don't need every field, you can use +#' @param filterByFormula String. Use a formula to filter results. See \href{airtable docs}{https://support.airtable.com/hc/en-us/articles/223247187-How-to-sort-filter-or-retrieve-ordered-records-in-the-API} #' this parameter to reduce the amount of data transferred. #' @return A data frame with records or a list with record details if #' \code{record_id} is specified. @@ -82,6 +83,7 @@ air_get <- function(base, table_name, fields = NULL, sortField = NULL, sortDirection = NULL, + filterByFormula = NULL, combined_result = TRUE) { search_path <- table_name @@ -94,7 +96,7 @@ air_get <- function(base, table_name, # append parameters to URL: param_list <- as.list(environment())[c( - "limit", "offset", "view", "sortField", "sortDirection")] + "limit", "offset", "view", "sortField", "sortDirection","filterByFormula")] param_list <- param_list[!sapply(param_list, is.null)] if(!is.null(fields)) { param_list <- c(param_list, list_params(x = fields, par_name = "fields")) @@ -103,6 +105,7 @@ air_get <- function(base, table_name, request_url <- httr::modify_url(request_url, query = param_list) request_url <- gsub(pattern = "fields=",replacement = "fields%5B%5D=",x = request_url) + print(request_url) # call service: res <- httr::GET( url = request_url, diff --git a/man/air_get.Rd b/man/air_get.Rd index 81b38cd..c5514f2 100644 --- a/man/air_get.Rd +++ b/man/air_get.Rd @@ -14,6 +14,7 @@ air_get( fields = NULL, sortField = NULL, sortDirection = NULL, + filterByFormula = NULL, combined_result = TRUE ) } @@ -34,14 +35,16 @@ call. Note that this is represented by a record ID, not a numerical offset.} \item{view}{(optional) The name or ID of the view} \item{fields}{(optional) Only data for fields whose names are in this list -will be included in the records. If you don't need every field, you can use -this parameter to reduce the amount of data transferred.} +will be included in the records. If you don't need every field, you can use} \item{sortField}{(optional) The field name to use for sorting} \item{sortDirection}{(optional) "asc" or "desc". The sort order in which the records will be returned. Defaults to asc.} +\item{filterByFormula}{String. Use a formula to filter results. See \href{airtable docs}{https://support.airtable.com/hc/en-us/articles/223247187-How-to-sort-filter-or-retrieve-ordered-records-in-the-API} +this parameter to reduce the amount of data transferred.} + \item{combined_result}{If TRUE (default) all data is returned in the same data. If FALSE table fields are returned in separate \code{fields} element.} } From ba7d1954202ae5c1f67cb3685ce2353acbfb53c7 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 6 Sep 2022 12:59:30 -0500 Subject: [PATCH 049/126] removed print statement from air_get --- DESCRIPTION | 2 +- R/airtabler.R | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 928f62b..f259308 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.1.8 +Version: 0.1.9 Date: 2022-08-23 Author: Darko Bergant Maintainer: Collin Schwantes diff --git a/R/airtabler.R b/R/airtabler.R index 4c699dd..ea46cee 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -105,7 +105,7 @@ air_get <- function(base, table_name, request_url <- httr::modify_url(request_url, query = param_list) request_url <- gsub(pattern = "fields=",replacement = "fields%5B%5D=",x = request_url) - print(request_url) + #print(request_url) # call service: res <- httr::GET( url = request_url, From a02513b7d2ae39b757b1a8d1717e04cc924313a2 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 18 Oct 2022 08:25:14 +0200 Subject: [PATCH 050/126] updated air_get_json function --- R/air_get_json.R | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/R/air_get_json.R b/R/air_get_json.R index 81deb8a..9fd74dc 100644 --- a/R/air_get_json.R +++ b/R/air_get_json.R @@ -83,25 +83,27 @@ fetch_all_json <- function(base, table_name, ...) { offset <- airtabler::get_offset(air_parse(out[[length(out)]])) } - ## now we have a list of json text, paste together - # remove offset attributes and object delimiters {} - - # flatten list - out_flat <- purrr::flatten(out) - # paste collapse with delimiter - json_delimited <- paste(out_flat,collapse = "##########") - # remove delimiter and unnecessary json - # gsub behind ahead - json_open_preceeding_obj <- gsub(pattern = "\\],\"offset\":\"[:alnum:]{17,}/[:alnum:]{17,}\"}(?<##########)", - replacement = ",", - x = json_delimited) - # gsub look head - json_open_following_obj <- gsub(pattern = "(?<=##########)],{\"records\"\\:\\[", - replacement = "", - x = json_delimited) - - out <- dplyr::bind_rows(out) - cbind(id = out$id, out$fields, createdTime = out$createdTime, - stringsAsFactors = FALSE) + ## map over json text and remove offset attribute + out_json <- purrr::map_chr(out, function(x){ + + x_drop_offset <- gsub( '\\],\"offset\\":\\"\\w{1,}/\\w{1,}\\"}$',"",x) + + }) + + # drop initial json section in all but first record + out_json_2 <- out_json + + for(i in 1:length(out_json)){ + if(i > 1){ + out_json_2[i] <- gsub("\\{\"records\"\\:\\[",",",out_json[i]) + } + next() + } + + # make single json string + json_delimited <- paste(out_json_2,collapse = "") + + return(json_delimited) + } } From 30729ba4432875c90abf808979bde0d9c80ae4e6 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 18 Oct 2022 08:25:53 +0200 Subject: [PATCH 051/126] removed air_get_json ref from air_dump --- R/air_dump.R | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index b2e5ea4..1f6bfa5 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -428,14 +428,12 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) } -### Get just the json to preserve the structure -- essentially modified air_get - - - ### extract_base - returns a named list #' Dump all tables from a base into json files #' +#' Essentially air_get without converting to Rs +#' #' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' #' @param metadata Data.frame.Data frame with structural metadata - describes relationship between tables and fields. #' @param description Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. From 218b51cc6c799142b9b9045005ddce5b86ca2e68 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 31 Oct 2022 16:23:12 -0600 Subject: [PATCH 052/126] updated description --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 19ed3d4..5348c04 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.1.7 +Version: 0.2.0 Date: 2017-09-17 Author: Darko Bergant Maintainer: Darko Bergant From b6d7895b7a97690c3cde218f81d770ce12a41d21 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 31 Oct 2022 17:28:50 -0600 Subject: [PATCH 053/126] added json export functions --- DESCRIPTION | 2 +- NAMESPACE | 3 +++ R/air_dump.R | 1 + R/air_get_json.R | 14 +++++++++- man/air_dump_to_json.Rd | 25 ++++++++++++++++++ man/air_get_json.Rd | 57 +++++++++++++++++++++++++++++++++++++++++ man/fetch_all_json.Rd | 18 +++++++++++++ 7 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 man/air_dump_to_json.Rd create mode 100644 man/air_get_json.Rd create mode 100644 man/fetch_all_json.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 48228d6..00ece35 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -23,4 +23,4 @@ Imports: snakecase, tidyselect, rlang -RoxygenNote: 7.2.0 +RoxygenNote: 7.2.1 diff --git a/NAMESPACE b/NAMESPACE index 42a7868..b5f4005 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -4,11 +4,13 @@ S3method(print,airtable.base) export(air_delete) export(air_dump) export(air_dump_to_csv) +export(air_dump_to_json) export(air_generate_base_description) export(air_generate_metadata) export(air_get) export(air_get_attachments) export(air_get_base_description_from_table) +export(air_get_json) export(air_get_metadata_from_table) export(air_get_schema) export(air_insert) @@ -20,6 +22,7 @@ export(air_update) export(air_update_data_frame) export(airtable) export(fetch_all) +export(fetch_all_json) export(flatten_col_to_chr) export(get_offset) export(get_unique_field_values) diff --git a/R/air_dump.R b/R/air_dump.R index 1f6bfa5..b9315b6 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -438,6 +438,7 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) #' @param metadata Data.frame.Data frame with structural metadata - describes relationship between tables and fields. #' @param description Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. #' Can be left as NULL if base already contains a table called description. +#' @param add_missing_fields Logical. If true add in missing fields #' #' @return List of data.frames. All tables from metadata plus the #' description and metadata tables. diff --git a/R/air_get_json.R b/R/air_get_json.R index 9fd74dc..f3f7860 100644 --- a/R/air_get_json.R +++ b/R/air_get_json.R @@ -22,7 +22,8 @@ #' this parameter to reduce the amount of data transferred. #' @return A data frame with records or a list with record details if #' \code{record_id} is specified. -#' @export +#' +#' @export air_get_json air_get_json <- function(base, table_name, record_id = NULL, limit = NULL, @@ -68,6 +69,17 @@ air_get_json <- function(base, table_name, } +#' Get the full outputs of a table as single json object +#' +#' @param base String. Base ID +#' @param table_name String. Table name +#' @param ... additional parameters to pass to air_get_json +#' +#' @return +#' @export fetch_all_json +#' +#' @examples +#' fetch_all_json <- function(base, table_name, ...) { out <- list() out[[1]] <- air_get_json(base, table_name, combined_result = FALSE,...) diff --git a/man/air_dump_to_json.Rd b/man/air_dump_to_json.Rd new file mode 100644 index 0000000..90467d4 --- /dev/null +++ b/man/air_dump_to_json.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_dump_to_json} +\alias{air_dump_to_json} +\title{Dump all tables from a base into json files} +\usage{ +air_dump_to_json(base, metadata, description = NULL, add_missing_fields = TRUE) +} +\arguments{ +\item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} + +\item{metadata}{Data.frame.Data frame with structural metadata - describes relationship between tables and fields.} + +\item{description}{Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. +Can be left as NULL if base already contains a table called description.} + +\item{add_missing_fields}{Logical. If true add in missing fields} +} +\value{ +List of data.frames. All tables from metadata plus the +description and metadata tables. +} +\description{ +Essentially air_get without converting to Rs +} diff --git a/man/air_get_json.Rd b/man/air_get_json.Rd new file mode 100644 index 0000000..7bebb86 --- /dev/null +++ b/man/air_get_json.Rd @@ -0,0 +1,57 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_get_json.R +\name{air_get_json} +\alias{air_get_json} +\title{Get a list of records or retrieve a single record as JSON} +\usage{ +air_get_json( + base, + table_name, + record_id = NULL, + limit = NULL, + offset = NULL, + view = NULL, + fields = NULL, + sortField = NULL, + sortDirection = NULL, + combined_result = TRUE, + pretty = FALSE +) +} +\arguments{ +\item{base}{Airtable base} + +\item{table_name}{Table name} + +\item{record_id}{(optional) Use record ID argument to retrieve an existing +record details} + +\item{limit}{(optional) A limit on the number of records to be returned. +Limit can range between 1 and 100.} + +\item{offset}{(optional) Page offset returned by the previous list-records +call. Note that this is represented by a record ID, not a numerical offset.} + +\item{view}{(optional) The name or ID of the view} + +\item{fields}{(optional) Only data for fields whose names are in this list +will be included in the records. If you don't need every field, you can use} + +\item{sortField}{(optional) The field name to use for sorting} + +\item{sortDirection}{(optional) "asc" or "desc". The sort order in which the +records will be returned. Defaults to asc.} + +\item{combined_result}{If TRUE (default) all data is returned in the same data. +If FALSE table fields are returned in separate \code{fields} element.} + +\item{pretty}{Logical. Should JSON be returned in human readable form? +this parameter to reduce the amount of data transferred.} +} +\value{ +A data frame with records or a list with record details if + \code{record_id} is specified. +} +\description{ +Returns JSON objects from GET requests +} diff --git a/man/fetch_all_json.Rd b/man/fetch_all_json.Rd new file mode 100644 index 0000000..316efbc --- /dev/null +++ b/man/fetch_all_json.Rd @@ -0,0 +1,18 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_get_json.R +\name{fetch_all_json} +\alias{fetch_all_json} +\title{Get the full outputs of a table as single json object} +\usage{ +fetch_all_json(base, table_name, ...) +} +\arguments{ +\item{base}{String. Base ID} + +\item{table_name}{String. Table name} + +\item{...}{additional parameters to pass to air_get_json} +} +\description{ +Get the full outputs of a table as single json object +} From a13a58ecceff139aa1da56661762be9bf9ac6b94 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 31 Oct 2022 17:32:29 -0600 Subject: [PATCH 054/126] update version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 00ece35..1fbe33a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.0 +Version: 0.2.1 Date: 2017-09-17 Author: Darko Bergant Maintainer: Collin Schwantes From 0bef6224796776304bc52a3f978576daf7ca6e78 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 12 Dec 2022 15:17:12 -0600 Subject: [PATCH 055/126] updated readme to reference tokens and not API keys --- readme.rmd | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/readme.rmd b/readme.rmd index f5d7f7d..96c1dd8 100644 --- a/readme.rmd +++ b/readme.rmd @@ -21,8 +21,11 @@ devtools::install_github("ecohealthalliance/airtabler") graphical interface, your Airtable base will provide its own API to create, read, update, and destroy records. - [airtable.com/api](http://airtable.com/api) -## Get and store the API key -Generate the airtable API key from your [Airtable account](http://airtable.com/account) page. +## Get and store the API tokens + +** As of November 2022 Airtable recommends using scoped tokens. The personal access tokens can be used interchangeably with the now superseded Airtable API Key. + +Create appropriately [scoped personal access tokens](https://airtable.com/developers/web/guides/personal-access-tokens). __airtabler__ functions will read the API key from environment variable `AIRTABLE_API_KEY`. To start R session with the @@ -42,6 +45,8 @@ AIRTABLE_API_KEY=your_api_key_here To check where your home is, type `path.expand("~")` in your R console. +If you're frequently working across multiple bases, consider using [`gitcrypt`](https://ecohealthalliance.github.io/eha-ma-handbook/16-encryption.html#set-up-encryption-for-a-repo-that-did-not-previously-use-git-crypt.) and the [`dotenv` package](https://cran.r-project.org/web/packages/dotenv/dotenv.pdf) to securely manage multiple tokens. + ## Usage From c3f50f66969a7a5b8042c7514f93572484c0e793 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 12 Dec 2022 15:17:43 -0600 Subject: [PATCH 056/126] updated readme --- readme.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 2fe56ec..f0df7ca 100644 --- a/readme.md +++ b/readme.md @@ -20,8 +20,11 @@ devtools::install_github("ecohealthalliance/airtabler") graphical interface, your Airtable base will provide its own API to create, read, update, and destroy records. - [airtable.com/api](http://airtable.com/api) -## Get and store the API key -Generate the airtable API key from your [Airtable account](http://airtable.com/account) page. +## Get and store the API tokens + +** As of November 2022 Airtable recommends using scoped tokens. The personal access tokens can be used interchangeably with the now superseded Airtable API Key. + +Create appropriately [scoped personal access tokens](https://airtable.com/developers/web/guides/personal-access-tokens). __airtabler__ functions will read the API key from environment variable `AIRTABLE_API_KEY`. To start R session with the @@ -41,6 +44,8 @@ AIRTABLE_API_KEY=your_api_key_here To check where your home is, type `path.expand("~")` in your R console. +If you're frequently working across multiple bases, consider using [`gitcrypt`](https://ecohealthalliance.github.io/eha-ma-handbook/16-encryption.html#set-up-encryption-for-a-repo-that-did-not-previously-use-git-crypt.) and the [`dotenv` package](https://cran.r-project.org/web/packages/dotenv/dotenv.pdf) to securely manage multiple tokens. + ## Usage From 7dc16cce2ef37a490a1667af472a762faccdc78c Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 12 Dec 2022 15:18:46 -0600 Subject: [PATCH 057/126] updated example for clarity --- readme.md | 2 +- readme.rmd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index f0df7ca..70e05a1 100644 --- a/readme.md +++ b/readme.md @@ -36,7 +36,7 @@ usethis::edit_r_environ In .Renviron add the following: ``` -AIRTABLE_API_KEY=your_api_key_here +AIRTABLE_API_KEY=your_api_token_here ``` diff --git a/readme.rmd b/readme.rmd index 96c1dd8..9350563 100644 --- a/readme.rmd +++ b/readme.rmd @@ -37,7 +37,7 @@ usethis::edit_r_environ In .Renviron add the following: ``` -AIRTABLE_API_KEY=your_api_key_here +AIRTABLE_API_KEY=your_api_token_here ``` From beb2f8590437749e17f1e96eb23ad71f1592da03 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 12 Dec 2022 15:19:44 -0600 Subject: [PATCH 058/126] removed reference to api key --- readme.md | 2 +- readme.rmd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 70e05a1..4dcb39b 100644 --- a/readme.md +++ b/readme.md @@ -26,7 +26,7 @@ read, update, and destroy records. - [airtable.com/api](http://airtable.com/api Create appropriately [scoped personal access tokens](https://airtable.com/developers/web/guides/personal-access-tokens). -__airtabler__ functions will read the API key from +__airtabler__ functions will read the API token from environment variable `AIRTABLE_API_KEY`. To start R session with the initialized environvent variable create an `.Renviron` file in your home directory. diff --git a/readme.rmd b/readme.rmd index 9350563..eee641d 100644 --- a/readme.rmd +++ b/readme.rmd @@ -27,7 +27,7 @@ read, update, and destroy records. - [airtable.com/api](http://airtable.com/api Create appropriately [scoped personal access tokens](https://airtable.com/developers/web/guides/personal-access-tokens). -__airtabler__ functions will read the API key from +__airtabler__ functions will read the API token from environment variable `AIRTABLE_API_KEY`. To start R session with the initialized environvent variable create an `.Renviron` file in your home directory. From 81ab49dcdc5a6dd9dbf6a50ccfac46acb1fc9c39 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 15 Dec 2022 11:48:04 -0600 Subject: [PATCH 059/126] added download function and helper functions for working with data that was converted to CSV --- DESCRIPTION | 4 +- NAMESPACE | 2 + R/air_download_attachments.R | 83 +++++++++++++++ R/air_dump.R | 174 +++++++++++++++++++++----------- R/air_expand_csv_arrays.R | 51 ++++++++++ R/air_get_attachments.R | 18 +--- R/air_get_json.R | 2 +- R/airtabler.R | 2 +- R/fetch_all.R | 2 +- man/air_download_attachments.Rd | 26 +++++ man/air_dump.Rd | 22 +++- man/air_expand_csv_arrays.Rd | 42 ++++++++ man/air_update_data_frame.Rd | 3 + man/fetch_all.Rd | 2 +- man/fetch_all_json.Rd | 3 + 15 files changed, 352 insertions(+), 84 deletions(-) create mode 100644 R/air_download_attachments.R create mode 100644 R/air_expand_csv_arrays.R create mode 100644 man/air_download_attachments.Rd create mode 100644 man/air_expand_csv_arrays.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 1fbe33a..25c37c3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -11,6 +11,7 @@ Depends: License: MIT + file LICENSE URL: https://github.com/ecohealthalliance/airtabler LazyData: TRUE +Encoding: UTF-8 Imports: glue, dplyr, @@ -22,5 +23,6 @@ Imports: utils, snakecase, tidyselect, - rlang + rlang, + stringr RoxygenNote: 7.2.1 diff --git a/NAMESPACE b/NAMESPACE index b5f4005..3900371 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -2,9 +2,11 @@ S3method(print,airtable.base) export(air_delete) +export(air_download_attachments) export(air_dump) export(air_dump_to_csv) export(air_dump_to_json) +export(air_expand_csv_arrays) export(air_generate_base_description) export(air_generate_metadata) export(air_get) diff --git a/R/air_download_attachments.R b/R/air_download_attachments.R new file mode 100644 index 0000000..f2710ed --- /dev/null +++ b/R/air_download_attachments.R @@ -0,0 +1,83 @@ +#' Download Airtable file attachments +#' +#' Download an attachment stored in air tables. Returns original dataframe +#' with an additional field called attachment_file_paths. The attachment_file_paths +#' field is of class list so it can handle multiple attachments per record. +#' +#' @param x Data frame. Output from air_get or fetch_all. +#' @param field String. Name of field with file attachments in base +#' @param dir_name String. Where should files be downloaded to? +#' Will create the folder if it does not exist. Folders created are recursively. +#' @param ... reserved for additional arguments. +#' +#' @return Returns x with an additional field called attachment_file_paths +#' @export air_download_attachments +#' +#' @examples +air_download_attachments <- function(x, field, dir_name = "downloads",...){ + #browser() + + if(!is.data.frame(x)){ + rlang::abort("x is not a dataframe") + } + + if(!field %in% names(x)){ + error_msg <- glue::glue("{field} not found in names(x). Check the name of the column + used to store attachments in airtable") + + rlang::abort(error_msg) + } + + if(!is.list(x[,field])){ + error_msg <- glue::glue("{field} is not of class list. Verify the name of + the column used to store attachments in airtable") + rlang::abort(error_msg) + } + + ### subset to necessary records ---- + + # get files + xfield <- purrr::pluck(x,field) + + ### get files ---- + dir.create(path = dir_name,recursive = TRUE) + + xlist <- purrr::map(xfield, function(x){ + + if(is.null(x$url)){ + ID <- x$id + warning(sprintf("Record ID %s is null",ID)) + return(NULL) + } + + # prepending attachment id in case the file naming convention + # of the user does not preclude duplicate file names for files + # with different contents - e.g. original file generation was + # structured like sample_1234/fasta.file sample_1235/fasta.file + + dest <- sprintf("%s/%s_%s", dir_name,x$id,x$filename) + + a <- utils::download.file(url = x$url,destfile = dest) + print(a) + + return(dest) + }) + + down_load_message <- glue::glue("Files downloaded here {dir_name}") + + message(down_load_message) + + field_file_paths <- sprintf("%s_file_paths",field) + + x$file_path <- xlist + + # using dynamic names in case a base has multiple file attachment + # columns + x <- dplyr::rename(x,{{field_file_paths}} := file_path) + return(x) + +} + + + + diff --git a/R/air_dump.R b/R/air_dump.R index b9315b6..df881aa 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -184,12 +184,20 @@ air_generate_base_description <- function(title = NA,primary_contact= NA,email = #' @param description Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. #' Can be left as NULL if base already contains a table called description. #' @param add_missing_fields Logical. Should fields described in the metadata data.frame be added to corresponding tables? +#' @param download_attachments Logical. Should attached files be downloaded? +#' @param ... Additional arguments to pass to air_download_attachments +#' @param attachment_fields Optional. character vector. +#' What field(s) should files be downloaded from? Default is to download all fields +#' with type multipleAttachments in metadata. #' #' @return List of data.frames. All tables from metadata plus the #' description and metadata tables. -#' @export +#' @export air_dump +#' +#' @note To facilitate joining on ids, see purrr::as_vector for converting list type columns to vectors and +#' tidyr::unnest for expanding list columns. #' -air_dump <- function(base, metadata, description = NULL, add_missing_fields = TRUE){ +air_dump <- function(base, metadata, description = NULL, add_missing_fields = TRUE, download_attachments = TRUE, attachment_fields=NULL,...){ names(metadata) <- snakecase::to_snake_case(names(metadata)) @@ -246,6 +254,45 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR } } + ## download files from attachment fields + + if(download_attachments){ + + if(rlang::is_empty(attachment_fields) & !"field_type"%in% names(metadata)){ + rlang::abort("Unclear which fields contain attachments. + Either use the attachment_fields argument or + supply a metadata dataframe with field_types == 'multipleAttachments' for + fields that should be downloaded") + } + + ## get attachment fields + if(is.null(attachment_fields)){ + metadata_table <- metadata[metadata$table_name == x,c("field_name","field_type")] + + attachment_fields <-metadata_table |> + dplyr::filter(field_type == "multipleAttachments") |> + dplyr::pull(field_name) + + if(rlang::is_empty(attachment_fields)){ + rlang::inform("No fields of type multipleAttachment. No files to download") + return(x_table) + } + } + + # check for field in table + if(is.character(attachment_fields)){ + if(!any(attachment_fields %in% names(x_table))){ + return(x_table) + } + } + + ## build up attachment fields on x_table + for(af in attachment_fields){ + x_table <- air_download_attachments(x_table,field = af,...) + } + } + + return(x_table) }) @@ -337,6 +384,7 @@ flatten_col_to_chr <- function(data_frame){ if(is.null(list_element)){ list_element <- "" } + row_value<- sprintf('"%s"',paste(list_element,collapse = ",")) chr_col <- append(chr_col,row_value) @@ -357,6 +405,7 @@ flatten_col_to_chr <- function(data_frame){ } + data_frame[i] <- chr_col } } @@ -445,65 +494,65 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) #' @export #' air_dump_to_json <- function(base, metadata, description = NULL, add_missing_fields = TRUE){ -# -# names(metadata) <- snakecase::to_snake_case(names(metadata)) -# -# ## check for required fields -# required_fields <- c("table_name","field_name") -# -# if(!all(required_fields %in% names(metadata))){ -# stop(glue::glue("metadata table must contain the -# following fields: {required_fields}. Note -# that field names are converted to snakecase -# before check.")) -# } -# -# -# base_table_names <- unique(metadata$table_name) -# -# print(base_table_names) -# table_list <- base_table_names |> -# purrr::set_names() |> -# purrr::map(function(x){ -# ## get fields from str_metadata -# -# fields_exp <- metadata[metadata$table_name == x,"field_name"] -# -# ## pull table - add check for blank tables -# x_table <- airtabler::fetch_all(base,x) -# -# if(!is.data.frame(x_table)){ -# x_table <- data.frame(id = character()) -# } -# -# ## add in missing columns if any -# fields_obs <- names(x_table) -# -# # check if any discrepancy between metadata and table -# fields_diff <- set_diff(fields_exp,fields_obs) -# #browser() -# if(!is.null(fields_diff)){ -# # check for fields in obs not in exp - error -# obs_exp <- setdiff(fields_obs,fields_exp) -# ignore_fields <- c("id","createdTime") -# ignore_fields_pattern <- paste(ignore_fields,collapse = "|") -# if(length(obs_exp) != 0 & !all(obs_exp %in% ignore_fields)){ -# missing_fields <- obs_exp[!grepl(ignore_fields_pattern,obs_exp,ignore.case = FALSE)] -# missing_fields_glue <- paste(missing_fields, collapse = ", ") -# stop(glue::glue('The metadata table is missing the following fields from table {x}: -# {missing_fields_glue} -# Please update the metadata table.https://airtable.com/{base}')) -# } -# # check for fields in exp and not in obs - append unless frictionless -# if(add_missing_fields){ -# exp_obs <- setdiff(fields_exp,fields_obs) -# x_table[exp_obs] <- list(character(0)) -# } -# } -# -# return(x_table) -# -# }) + # + # names(metadata) <- snakecase::to_snake_case(names(metadata)) + # + # ## check for required fields + # required_fields <- c("table_name","field_name") + # + # if(!all(required_fields %in% names(metadata))){ + # stop(glue::glue("metadata table must contain the + # following fields: {required_fields}. Note + # that field names are converted to snakecase + # before check.")) + # } + # + # + # base_table_names <- unique(metadata$table_name) + # + # print(base_table_names) + # table_list <- base_table_names |> + # purrr::set_names() |> + # purrr::map(function(x){ + # ## get fields from str_metadata + # + # fields_exp <- metadata[metadata$table_name == x,"field_name"] + # + # ## pull table - add check for blank tables + # x_table <- airtabler::fetch_all(base,x) + # + # if(!is.data.frame(x_table)){ + # x_table <- data.frame(id = character()) + # } + # + # ## add in missing columns if any + # fields_obs <- names(x_table) + # + # # check if any discrepancy between metadata and table + # fields_diff <- set_diff(fields_exp,fields_obs) + # #browser() + # if(!is.null(fields_diff)){ + # # check for fields in obs not in exp - error + # obs_exp <- setdiff(fields_obs,fields_exp) + # ignore_fields <- c("id","createdTime") + # ignore_fields_pattern <- paste(ignore_fields,collapse = "|") + # if(length(obs_exp) != 0 & !all(obs_exp %in% ignore_fields)){ + # missing_fields <- obs_exp[!grepl(ignore_fields_pattern,obs_exp,ignore.case = FALSE)] + # missing_fields_glue <- paste(missing_fields, collapse = ", ") + # stop(glue::glue('The metadata table is missing the following fields from table {x}: + # {missing_fields_glue} + # Please update the metadata table.https://airtable.com/{base}')) + # } + # # check for fields in exp and not in obs - append unless frictionless + # if(add_missing_fields){ + # exp_obs <- setdiff(fields_exp,fields_obs) + # x_table[exp_obs] <- list(character(0)) + # } + # } + # + # return(x_table) + # + # }) } @@ -524,3 +573,6 @@ air_dump_to_json <- function(base, metadata, description = NULL, add_missing_fie # }) # # } + + + diff --git a/R/air_expand_csv_arrays.R b/R/air_expand_csv_arrays.R new file mode 100644 index 0000000..61e6aaa --- /dev/null +++ b/R/air_expand_csv_arrays.R @@ -0,0 +1,51 @@ +#' Expand arrays stored in CSVs +#' +#' This function helps users work with airtable data that has been exported to CSVs. +#' Because airtable uses nested data structures (json arrays), the data must be +#' flattened to be stored in a csv. The standard way to store arrays in a csv +#' is to wrap the array in quotes and separate each item with commas. So an array +#' stored in a csv would look like "item 1,item 2,...,item n". This function will +#' convert arrays stored in csvs to either a list or a vector and removes the +#' surrounding quotes. +#' +#' +#' @param x Character. likely a vector or field in a dataframe. +#' @param simplify_to_vector Logical. Should expanded arrays be converted from +#' lists to vectors? For lists with multiple elements at a given position, the length of the output +#' may be greater than the length of the input. See [tidyr::unnest()] for expanding +#' list columns. +#' +#' +#' @return A vector or list of expanded arrays. +#' @export air_expand_csv_arrays +#' +#' @examples +#' +#' +#' # example vector data +#' x <- c("item 1,item 2,item 3","apple,orange,banana","1,2,3","") +#' +#' # to list +#' air_expand_csv_arrays(x) +#' +#' # to vector +#' air_expand_csv_arrays(x,simplify_to_vector = TRUE) +#' +#' +air_expand_csv_arrays <- function(x,simplify_to_vector = FALSE){ + + if(!is.character(x)){ + rlang::abort("x must be class character. x is class {class(x)}") + } + + split_x <- stringr::str_split(x,",") + + x_out <- purrr::map(.x = split_x,.f = stringr::str_replace_all,pattern = "^\"|\"$",replacement = "") + + if(simplify_to_vector){ + x_out <- purrr::as_vector(x_out,.type = "character") + } + + return(x_out) + +} diff --git a/R/air_get_attachments.R b/R/air_get_attachments.R index f2bf773..e0c8617 100644 --- a/R/air_get_attachments.R +++ b/R/air_get_attachments.R @@ -31,23 +31,7 @@ air_get_attachments <- function(base, table_name, field, download_file = FALSE, ### get files ---- if(download_file){ - dir.create(dir_name) - - xlist <- purrr::map(xfield, function(x){ - - if(is.null(x$url)){ - ID <- x$id - warning(sprintf("Record ID %s is null",ID)) - return(NULL) - } - - dest <- sprintf("%s/%s", dir_name,x$filename) - - utils::download.file(url = x$url,destfile = dest) - }) - - message("downloaded files in ./downloads") - + x <- air_download_attachments(x,field = field,dir_name = dir_name) } ### extract excel ---- diff --git a/R/air_get_json.R b/R/air_get_json.R index f3f7860..85d9762 100644 --- a/R/air_get_json.R +++ b/R/air_get_json.R @@ -75,7 +75,7 @@ air_get_json <- function(base, table_name, #' @param table_name String. Table name #' @param ... additional parameters to pass to air_get_json #' -#' @return +#' @return json as string #' @export fetch_all_json #' #' @examples diff --git a/R/airtabler.R b/R/airtabler.R index 2d47dde..fe8717d 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -477,7 +477,7 @@ air_insert_data_frame <- function(base, table_name, records,typecast) { #' @param record_ids Vector of strings. Records to be modified #' @param records Dataframe. Values to update #' -#' @return +#' @return Status of HTTP request #' @export #' #' @examples diff --git a/R/fetch_all.R b/R/fetch_all.R index 87cccb8..d911064 100644 --- a/R/fetch_all.R +++ b/R/fetch_all.R @@ -29,7 +29,7 @@ #' @export fetch_all #' #' @examples -#' # Each base a fully described API +#' # Each base has a fully described API #' app_id <- "appwlxIzmQx5njRtQ" # ID for the base we are fetching. #' # Note that you can pass a `view` argument to air_get or fetch_all to get only #' # a view of a table (say, only validated records, or some other filtered view), diff --git a/man/air_download_attachments.Rd b/man/air_download_attachments.Rd new file mode 100644 index 0000000..b4aad82 --- /dev/null +++ b/man/air_download_attachments.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_download_attachments.R +\name{air_download_attachments} +\alias{air_download_attachments} +\title{Download Airtable file attachments} +\usage{ +air_download_attachments(x, field, dir_name = "downloads", ...) +} +\arguments{ +\item{x}{Data frame. Output from air_get or fetch_all.} + +\item{field}{String. Name of field with file attachments in base} + +\item{dir_name}{String. Where should files be downloaded to? +Will create the folder if it does not exist. Folders created are recursively.} + +\item{...}{reserved for additional arguments.} +} +\value{ +Returns x with an additional field called attachment_file_paths +} +\description{ +Download an attachment stored in air tables. Returns original dataframe +with an additional field called attachment_file_paths. The attachment_file_paths +field is of class list so it can handle multiple attachments per record. +} diff --git a/man/air_dump.Rd b/man/air_dump.Rd index 4cfdc67..12ba234 100644 --- a/man/air_dump.Rd +++ b/man/air_dump.Rd @@ -4,7 +4,15 @@ \alias{air_dump} \title{Dump all tables from a base into R} \usage{ -air_dump(base, metadata, description = NULL, add_missing_fields = TRUE) +air_dump( + base, + metadata, + description = NULL, + add_missing_fields = TRUE, + download_attachments = TRUE, + attachment_fields = NULL, + ... +) } \arguments{ \item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} @@ -15,6 +23,14 @@ air_dump(base, metadata, description = NULL, add_missing_fields = TRUE) Can be left as NULL if base already contains a table called description.} \item{add_missing_fields}{Logical. Should fields described in the metadata data.frame be added to corresponding tables?} + +\item{download_attachments}{Logical. Should attached files be downloaded?} + +\item{attachment_fields}{Optional. character vector. +What field(s) should files be downloaded from? Default is to download all fields +with type multipleAttachments in metadata.} + +\item{...}{Additional arguments to pass to air_download_attachments} } \value{ List of data.frames. All tables from metadata plus the @@ -23,3 +39,7 @@ description and metadata tables. \description{ Dump all tables from a base into R } +\note{ +To facilitate joining on ids, see purrr::as_vector for converting list type columns to vectors and +tidyr::unnest for expanding list columns. +} diff --git a/man/air_expand_csv_arrays.Rd b/man/air_expand_csv_arrays.Rd new file mode 100644 index 0000000..5ab6746 --- /dev/null +++ b/man/air_expand_csv_arrays.Rd @@ -0,0 +1,42 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_expand_csv_arrays.R +\name{air_expand_csv_arrays} +\alias{air_expand_csv_arrays} +\title{Expand arrays stored in CSVs} +\usage{ +air_expand_csv_arrays(x, simplify_to_vector = FALSE) +} +\arguments{ +\item{x}{Character. likely a vector or field in a dataframe.} + +\item{simplify_to_vector}{Logical. Should expanded arrays be converted from +lists to vectors? For lists with multiple elements at a given position, the length of the output +may be greater than the length of the input. See [tidyr::unnest()] for expanding +list columns.} +} +\value{ +A vector or list of expanded arrays. +} +\description{ +This function helps users work with airtable data that has been exported to CSVs. +Because airtable uses nested data structures (json arrays), the data must be +flattened to be stored in a csv. The standard way to store arrays in a csv +is to wrap the array in quotes and separate each item with commas. So an array +stored in a csv would look like "item 1,item 2,...,item n". This function will +convert arrays stored in csvs to either a list or a vector and removes the +surrounding quotes. +} +\examples{ + + +# example vector data +x <- c("item 1,item 2,item 3","apple,orange,banana","1,2,3","") + +# to list +air_expand_csv_arrays(x) + +# to vector +air_expand_csv_arrays(x,simplify_to_vector = TRUE) + + +} diff --git a/man/air_update_data_frame.Rd b/man/air_update_data_frame.Rd index 375cd71..e736380 100644 --- a/man/air_update_data_frame.Rd +++ b/man/air_update_data_frame.Rd @@ -15,6 +15,9 @@ air_update_data_frame(base, table_name, record_ids, records) \item{records}{Dataframe. Values to update} } +\value{ +Status of HTTP request +} \description{ Updates the values in a table by overwriting their current contents. } diff --git a/man/fetch_all.Rd b/man/fetch_all.Rd index 320b895..c70b4af 100644 --- a/man/fetch_all.Rd +++ b/man/fetch_all.Rd @@ -41,7 +41,7 @@ file. } \examples{ -# Each base a fully described API +# Each base has a fully described API app_id <- "appwlxIzmQx5njRtQ" # ID for the base we are fetching. # Note that you can pass a `view` argument to air_get or fetch_all to get only # a view of a table (say, only validated records, or some other filtered view), diff --git a/man/fetch_all_json.Rd b/man/fetch_all_json.Rd index 316efbc..2591cac 100644 --- a/man/fetch_all_json.Rd +++ b/man/fetch_all_json.Rd @@ -13,6 +13,9 @@ fetch_all_json(base, table_name, ...) \item{...}{additional parameters to pass to air_get_json} } +\value{ +json as string +} \description{ Get the full outputs of a table as single json object } From 2998f9e7f7128f34672f32daf06e580af7086b07 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 15 Dec 2022 16:34:07 -0600 Subject: [PATCH 060/126] Update DESCRIPTION --- DESCRIPTION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 25c37c3..45d6fe7 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.1 -Date: 2017-09-17 +Version: 0.2.2 +Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes Description: Fork from Darko Bergant's package. Provides access to the Airtable (airtable.com) API. From 177d99dd799b320d3082be72f459de3d35c0ee2d Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 22 Dec 2022 14:06:00 -0600 Subject: [PATCH 061/126] added options to get_metadata that add in record id fields and allow users to set field_names to snakecase --- R/air_dump.R | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/R/air_dump.R b/R/air_dump.R index df881aa..ab230bc 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -46,11 +46,14 @@ set_diff <- function(x,y){ #' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' #' @param table_name String. Name of structural metadata table - the metadata that #' describes how tables and fields fit together. +#' @param add_id_field Logical. If true, an "id" field is added to each table +#' @param field_names_to_snake_case Logical. If true, values in the field_names +#' column are converted to snake_case #' #' @return data.frame with metadata table #' @export #' -air_get_metadata_from_table <- function(base, table_name){ +air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, field_names_to_snake_case = TRUE){ # get structural metadata table str_metadata <- airtabler::fetch_all(base,table_name) ## check for table_name, field_name @@ -64,6 +67,22 @@ air_get_metadata_from_table <- function(base, table_name){ before check.")) } + ## make field names snake_case + if(snakecase){ + str_metadata$field_names <- snakecase::to_snake_case(str_metadata$field_names) + } + + ## add id field to all tables + if(add_id_field){ + + tables <- dplyr::distinct(.data = str_metadata,table_name,.keep_all = TRUE) + tables$field_desc <- "unique id assigned by airtable" + tables$field_type <- "singleLineText" + + str_metadata <- rbind(str_metadata,tables) + + } + return(str_metadata) } From 6582a1d37ae4d36aa8483aa80c5d91bf22042eba Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 20 Jan 2023 09:03:18 -0600 Subject: [PATCH 062/126] added snakecase option to airdump so names are consistent across metadata and outputs --- R/air_dump.R | 21 +++++++++++++++------ R/fetch_all.R | 6 +++--- man/air_dump.Rd | 4 ++++ man/air_get_metadata_from_table.Rd | 12 +++++++++++- man/fetch_all.Rd | 8 ++++---- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index ab230bc..0441414 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -47,13 +47,13 @@ set_diff <- function(x,y){ #' @param table_name String. Name of structural metadata table - the metadata that #' describes how tables and fields fit together. #' @param add_id_field Logical. If true, an "id" field is added to each table -#' @param field_names_to_snake_case Logical. If true, values in the field_names +#' @param field_names_to_snakecase Logical. If true, values in the field_names #' column are converted to snake_case #' #' @return data.frame with metadata table #' @export #' -air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, field_names_to_snake_case = TRUE){ +air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, field_names_to_snakecase = TRUE){ # get structural metadata table str_metadata <- airtabler::fetch_all(base,table_name) ## check for table_name, field_name @@ -68,8 +68,8 @@ air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, f } ## make field names snake_case - if(snakecase){ - str_metadata$field_names <- snakecase::to_snake_case(str_metadata$field_names) + if(field_names_to_snakecase){ + str_metadata$field_name <- snakecase::to_snake_case(str_metadata$field_name) } ## add id field to all tables @@ -78,6 +78,8 @@ air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, f tables <- dplyr::distinct(.data = str_metadata,table_name,.keep_all = TRUE) tables$field_desc <- "unique id assigned by airtable" tables$field_type <- "singleLineText" + tables$field_id <- NA + tables$field_opt_name <- NA str_metadata <- rbind(str_metadata,tables) @@ -208,6 +210,8 @@ air_generate_base_description <- function(title = NA,primary_contact= NA,email = #' @param attachment_fields Optional. character vector. #' What field(s) should files be downloaded from? Default is to download all fields #' with type multipleAttachments in metadata. +#' @param field_names_to_snakecase Logical. Should field names be +#' converted to snake case? #' #' @return List of data.frames. All tables from metadata plus the #' description and metadata tables. @@ -216,7 +220,7 @@ air_generate_base_description <- function(title = NA,primary_contact= NA,email = #' @note To facilitate joining on ids, see purrr::as_vector for converting list type columns to vectors and #' tidyr::unnest for expanding list columns. #' -air_dump <- function(base, metadata, description = NULL, add_missing_fields = TRUE, download_attachments = TRUE, attachment_fields=NULL,...){ +air_dump <- function(base, metadata, description = NULL, add_missing_fields = TRUE, download_attachments = TRUE, attachment_fields=NULL, field_names_to_snakecase = TRUE,...){ names(metadata) <- snakecase::to_snake_case(names(metadata)) @@ -237,6 +241,7 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR table_list <- base_table_names |> purrr::set_names() |> purrr::map(function(x){ + ## get fields from str_metadata fields_exp <- metadata[metadata$table_name == x,"field_name"] @@ -248,6 +253,10 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR x_table <- data.frame(id = character()) } + if(field_names_to_snakecase){ + names(x_table) <- snakecase::to_snake_case(names(x_table)) + } + ## add in missing columns if any fields_obs <- names(x_table) @@ -257,7 +266,7 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR if(!is.null(fields_diff)){ # check for fields in obs not in exp - error obs_exp <- setdiff(fields_obs,fields_exp) - ignore_fields <- c("id","createdTime") + ignore_fields <- c("id","createdTime","created_time") ignore_fields_pattern <- paste(ignore_fields,collapse = "|") if(length(obs_exp) != 0 & !all(obs_exp %in% ignore_fields)){ missing_fields <- obs_exp[!grepl(ignore_fields_pattern,obs_exp,ignore.case = FALSE)] diff --git a/R/fetch_all.R b/R/fetch_all.R index d911064..cc60363 100644 --- a/R/fetch_all.R +++ b/R/fetch_all.R @@ -30,12 +30,12 @@ #' #' @examples #' # Each base has a fully described API -#' app_id <- "appwlxIzmQx5njRtQ" # ID for the base we are fetching. +#' # app_id <- "appVjIfAo8AJlfTkx" # ID for the base we are fetching. #' # Note that you can pass a `view` argument to air_get or fetch_all to get only #' # a view of a table (say, only validated records, or some other filtered view), #' # e.g., -#' # bats <- fetch_all(app_id, "Bat", view = "Validated Records") -#' talks <- fetch_all(app_id, "TALKS") +#' # bats <- fetch_all(app_id, "images", view = "Status View") +#' # talks <- fetch_all(app_id, "images") #' fetch_all <- function(base, table_name, ...) { out <- list() diff --git a/man/air_dump.Rd b/man/air_dump.Rd index 12ba234..590abd5 100644 --- a/man/air_dump.Rd +++ b/man/air_dump.Rd @@ -11,6 +11,7 @@ air_dump( add_missing_fields = TRUE, download_attachments = TRUE, attachment_fields = NULL, + field_names_to_snakecase = TRUE, ... ) } @@ -30,6 +31,9 @@ Can be left as NULL if base already contains a table called description.} What field(s) should files be downloaded from? Default is to download all fields with type multipleAttachments in metadata.} +\item{field_names_to_snakecase}{Logical. Should field names be +converted to snake case?} + \item{...}{Additional arguments to pass to air_download_attachments} } \value{ diff --git a/man/air_get_metadata_from_table.Rd b/man/air_get_metadata_from_table.Rd index 44a34dc..9d893dc 100644 --- a/man/air_get_metadata_from_table.Rd +++ b/man/air_get_metadata_from_table.Rd @@ -4,13 +4,23 @@ \alias{air_get_metadata_from_table} \title{Pull the metadata table from airtable} \usage{ -air_get_metadata_from_table(base, table_name) +air_get_metadata_from_table( + base, + table_name, + add_id_field = TRUE, + field_names_to_snakecase = TRUE +) } \arguments{ \item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} \item{table_name}{String. Name of structural metadata table - the metadata that describes how tables and fields fit together.} + +\item{add_id_field}{Logical. If true, an "id" field is added to each table} + +\item{field_names_to_snakecase}{Logical. If true, values in the field_names +column are converted to snake_case} } \value{ data.frame with metadata table diff --git a/man/fetch_all.Rd b/man/fetch_all.Rd index c70b4af..3f9e4c5 100644 --- a/man/fetch_all.Rd +++ b/man/fetch_all.Rd @@ -41,12 +41,12 @@ file. } \examples{ -# Each base has a fully described API -app_id <- "appwlxIzmQx5njRtQ" # ID for the base we are fetching. +# Each base has a fully described API +# app_id <- "appVjIfAo8AJlfTkx" # ID for the base we are fetching. # Note that you can pass a `view` argument to air_get or fetch_all to get only # a view of a table (say, only validated records, or some other filtered view), # e.g., -# bats <- fetch_all(app_id, "Bat", view = "Validated Records") -talks <- fetch_all(app_id, "TALKS") +# bats <- fetch_all(app_id, "images", view = "Status View") +# talks <- fetch_all(app_id, "images") } From fc41e85b83cdad682da47e440a309205762910d5 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 20 Jan 2023 14:36:20 -0600 Subject: [PATCH 063/126] updated readme to include warning about using metadata api --- readme.md | 6 ++++++ readme.rmd | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/readme.md b/readme.md index 4dcb39b..c38637e 100644 --- a/readme.md +++ b/readme.md @@ -47,6 +47,12 @@ To check where your home is, type `path.expand("~")` in your R console. If you're frequently working across multiple bases, consider using [`gitcrypt`](https://ecohealthalliance.github.io/eha-ma-handbook/16-encryption.html#set-up-encryption-for-a-repo-that-did-not-previously-use-git-crypt.) and the [`dotenv` package](https://cran.r-project.org/web/packages/dotenv/dotenv.pdf) to securely manage multiple tokens. +### Using the metadata API + +In order to use the metadata API, a personal access token or OAuth integration +must be used. User API keys are not supported. See the [airtable guide](https://airtable.com/developers/web/api/get-base-schema) for more +information. + ## Usage Create airtable base object: diff --git a/readme.rmd b/readme.rmd index eee641d..ad7f16a 100644 --- a/readme.rmd +++ b/readme.rmd @@ -48,6 +48,12 @@ To check where your home is, type `path.expand("~")` in your R console. If you're frequently working across multiple bases, consider using [`gitcrypt`](https://ecohealthalliance.github.io/eha-ma-handbook/16-encryption.html#set-up-encryption-for-a-repo-that-did-not-previously-use-git-crypt.) and the [`dotenv` package](https://cran.r-project.org/web/packages/dotenv/dotenv.pdf) to securely manage multiple tokens. +### Using the metadata API + +In order to use the metadata API, a personal access token or OAuth integration +must be used. User API keys are not supported. See the [airtable guide](https://airtable.com/developers/web/api/get-base-schema) for more +information. + ## Usage Create airtable base object: From 9ff797a89f6d9bb5691f9033ea0c8deb6527897f Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 23 Jan 2023 09:58:06 -0600 Subject: [PATCH 064/126] updated get schema and added placeholders for api metdata --- R/air_dump.R | 29 +++++++++++++++++++++++++++++ R/air_get_schema.R | 16 ++++++++-------- R/airtabler.R | 3 +++ man/air_get_schema.Rd | 9 ++++----- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index 0441414..e751b62 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -36,6 +36,35 @@ set_diff <- function(x,y){ return(diff) } +# add the metadata table to the base +air_insert_metadata_table <- function(base){ + +} + +# add the description table to the base + +air_insert_description_table <- function(base){ + +} + +# pull data from api and populate metadata table +air_get_metadata_from_api <- function(base){ + + # create metadata table skeleton + md_df <- data.frame(field_name = "", + table_name = "", + field_desc = "", + field_type = "", + field_id = "", + table_id = "", + field_opts = "") + # get base schema + + # parse base schema to populate metadata table + + # insert metadata table +} + #' Pull the metadata table from airtable #' #' For information about creating metadata tables in your base see the diff --git a/R/air_get_schema.R b/R/air_get_schema.R index 1c13fe2..f960bd4 100644 --- a/R/air_get_schema.R +++ b/R/air_get_schema.R @@ -1,11 +1,10 @@ -#' Get base schema - enterprise only +#' Get base schema #' -#' Metadata API currently only available via enterprise accounts. -#' Get the schema for the tables in a base. +#' Get the schema for the tables in a base. This is a wrapper for the api call +#' Get base schema. #' #' @section Using Metadata API: -#' The meta data api is not accepting new requests for client secrets as of -#' 06 July 2021. Will update when metadata api becomes available. +#' Metadata api is currently available to all users. #' #' @param base Airtable base ID #' @param ... additional paramters @@ -21,15 +20,16 @@ air_get_schema <- function(base,...){ res <- httr::GET( request_url, httr::add_headers( - Authorization = paste("Bearer", air_api_key()), - "X-Airtable-Client-Secret" = air_secret_key() + Authorization = paste("Bearer", air_api_key()) ) ) air_validate(res) # may need a new air_parse function - schema <- jsonlite::fromJSON(res) + res_content <- httr::content(res,as = "text") + + schema <- jsonlite::fromJSON(res_content) return(schema) } diff --git a/R/airtabler.R b/R/airtabler.R index fe8717d..4b35170 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -31,6 +31,8 @@ NULL air_url <- "https://api.airtable.com/v0" air_meta_url <- "https://api.airtable.com/v0/meta/bases" +# consider consolidating keys as the API key is now a token that can access +# the metadata api air_api_key <- function() { key <- Sys.getenv("AIRTABLE_API_KEY") if(key == "") { @@ -39,6 +41,7 @@ air_api_key <- function() { key } +# consider air_secret_key <- function(){ key <- Sys.getenv("AIRTABLE_SECRET_KEY") if(key == "") { diff --git a/man/air_get_schema.Rd b/man/air_get_schema.Rd index 43039d3..ab7a0e3 100644 --- a/man/air_get_schema.Rd +++ b/man/air_get_schema.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/air_get_schema.R \name{air_get_schema} \alias{air_get_schema} -\title{Get base schema - enterprise only} +\title{Get base schema} \usage{ air_get_schema(base, ...) } @@ -15,12 +15,11 @@ air_get_schema(base, ...) list of schema } \description{ -Metadata API currently only available via enterprise accounts. -Get the schema for the tables in a base. +Get the schema for the tables in a base. This is a wrapper for the api call +Get base schema. } \section{Using Metadata API}{ -The meta data api is not accepting new requests for client secrets as of -06 July 2021. Will update when metadata api becomes available. +Metadata api is currently available to all users. } From 32b64f0b0cdc027d7ab5fd6eee1972dc7bf548d1 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 10 Feb 2023 10:10:40 -0600 Subject: [PATCH 065/126] added function to get metadata from api --- R/air_dump.R | 103 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 85 insertions(+), 18 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index e751b62..f46a787 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -37,33 +37,18 @@ set_diff <- function(x,y){ } # add the metadata table to the base -air_insert_metadata_table <- function(base){ +air_insert_metadata_table <- function(base,meta_data,table_name = "Meta Data"){ + } # add the description table to the base -air_insert_description_table <- function(base){ +air_insert_description_table <- function(base,description){ } -# pull data from api and populate metadata table -air_get_metadata_from_api <- function(base){ - - # create metadata table skeleton - md_df <- data.frame(field_name = "", - table_name = "", - field_desc = "", - field_type = "", - field_id = "", - table_id = "", - field_opts = "") - # get base schema - - # parse base schema to populate metadata table - # insert metadata table -} #' Pull the metadata table from airtable #' @@ -118,6 +103,88 @@ air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, f } +# pull data from api and populate metadata table +air_generate_metadata_from_api <- function(base, + metadata_table_name = "Meta Data", + include_metadata_table = FALSE){ + + # get base schema + schema <- air_get_schema(base) + + tables_df <- schema$tables + + if(!include_metadata_table){ + tables_df <- tables_df[stringr::str_detect(tables_df$name, + pattern = metadata_table_name, + negate = TRUE),] + } + + # parse base schema to populate metadata table + + #split by table id to parse with purrr + schema_list <- split(tables_df,f = tables_df$id) + + metadata_df <- purrr::map_dfr(schema_list, function(x){ + + # create metadata table skeleton + fields_df <-x$fields[[1]] + + fields_df$choices <- "" + fields_df$linkedTableID <- "" + + # get flattened choice names + if(!rlang::is_empty(fields_df$options$choices)){ + fields_df$choices <- purrr::map_chr(fields_df$options$choices,function(x){ + if(is.null(x)){ + return("") + } else { + return(paste(x$name,collapse = ", ")) + } + }) + } + + # get linked table id + if(!rlang::is_empty(fields_df$options$linkedTableId)){ + fields_df$linkedTableID <- fields_df$options$linkedTableId + } + + + fields_df <- fields_df |> + dplyr::mutate(field_opts = + dplyr::case_when( + type == "multipleSelects" | type == "singleSelect" ~ choices, + type == "multipleRecordLinks" ~ linkedTableID, + TRUE ~ "" + ) + ) + + + + + + # check that descriptions arent empty + + if(rlang::is_empty(fields_df$description)){ + fields_df$description <- "" + } + + md_df <- data.frame(field_name = fields_df$name, + table_name = x$name, + field_desc = fields_df$description, + field_type = fields_df$type, + field_id = fields_df$id, + table_id = x$id, + field_opts = fields_df$field_opts, + primary_key = (x$primaryFieldId == fields_df$id) + ) + + }) + + return(metadata_df) +} + +##air_insert + #' Generated Metadata from table names #' #' Generates a structural metadata table - the metadata that From 653c2055b88945253ca32ffc332471c31e7d2b82 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 14 Feb 2023 14:42:58 -0600 Subject: [PATCH 066/126] added function to create metadata table --- R/air_dump.R | 120 ++++++++++++++++++++++++++++++++--- R/air_metadata_api.R | 146 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+), 9 deletions(-) create mode 100644 R/air_metadata_api.R diff --git a/R/air_dump.R b/R/air_dump.R index f46a787..e036752 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -37,17 +37,118 @@ set_diff <- function(x,y){ } # add the metadata table to the base -air_insert_metadata_table <- function(base,meta_data,table_name = "Meta Data"){ +#' Create a new metadata table in the base +#' +#' @param base String. Base id +#' @param meta_data Data frame. Contains metadata records. From air_generate_metadata* +#' @param table_name String. name of the metadata table. default is "Meta Data" +#' +#' @return List with outcome from creating the table and inserting the records +#' @export air_create_metadata_table +#' +#' @examples +air_create_metadata_table <- function(base,meta_data,table_name = "Meta Data"){ + + # check for meta data table + ## if exists, stop + schema <- air_get_schema("appVjIfAo8AJlfTkx") + + if(table_name %in% schema$tables$name){ + msg <- glue::glue("{table_name} already exists in the base {base}. + Please use air_update_metadata_table to update the metadata table + or delete the table and re-run the function") + stop(msg) + } + + + # create fields_df + # add description for standard names + description <- NA + type <- "singleLineText" + + # + if(setequal(names(meta_data), c("field_name", "table_name", "field_desc", + "field_type", "field_id", "table_id", + "field_opts", "primary_key"))){ + + # create description object + description <- c("https://schema.org/name", + "https://schema.org/name", + "https://schema.org/description", + "https://schema.org/category", + "https://schema.org/identifier", + "https://schema.org/identifier", + "https://schema.org/option", + "https://schema.org/Boolean" + ) + + + } + + fields_df <- air_fields_df_template(name = names(meta_data), + description = description, + type = "singleLineText", + options = NA) + + # create list describing table + + table_list <- air_table_template(table_name = table_name, + description = "structural metadata for the base", + fields_df = fields_df) + # create table + outcome_create_table <- air_create_table(base, table_list) + + # insert data + + outcome_insert_data <- tryCatch( + air_insert_data_frame(base = base,table_name = table_name,records = meta_data), + error=function(cond) { + + warning(cond) + + return("data not inserted") + }) + + if(is.character(outcome_insert_data)){ + stop("Table created but data not inserted. Check field types then use + air_insert_data_frame or air_update_metadata_table to add metadata + records.") + } + + + return(list("create_table" = outcome_create_table, + "insert_data" = outcome_insert_data)) } # add the description table to the base -air_insert_description_table <- function(base,description){ +air_insert_description_table <- function(base,description, table_name = "Description"){ } +# update metadata table + +air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data"){ + + # check for Meta Data table + ## if no meta data table, stop + # pull down current meta data table + # compare with updated values + # update records + # create new records + # drop records no longer in meta data + +} + +# update description table +air_update_description_table <- function(base,description, table_name = "Description"){ + + # check for description + # update records + +} #' Pull the metadata table from airtable @@ -151,11 +252,11 @@ air_generate_metadata_from_api <- function(base, fields_df <- fields_df |> dplyr::mutate(field_opts = - dplyr::case_when( - type == "multipleSelects" | type == "singleSelect" ~ choices, - type == "multipleRecordLinks" ~ linkedTableID, - TRUE ~ "" - ) + dplyr::case_when( + type == "multipleSelects" | type == "singleSelect" ~ choices, + type == "multipleRecordLinks" ~ linkedTableID, + TRUE ~ "" + ) ) @@ -175,12 +276,13 @@ air_generate_metadata_from_api <- function(base, field_id = fields_df$id, table_id = x$id, field_opts = fields_df$field_opts, - primary_key = (x$primaryFieldId == fields_df$id) + primary_key = as.character(x$primaryFieldId == fields_df$id) ) + }) - return(metadata_df) +return(metadata_df) } ##air_insert diff --git a/R/air_metadata_api.R b/R/air_metadata_api.R new file mode 100644 index 0000000..27484ba --- /dev/null +++ b/R/air_metadata_api.R @@ -0,0 +1,146 @@ +#' Get base schema +#' +#' Get the schema for the tables in a base. This is a wrapper for the api call +#' Get base schema. +#' +#' @section Using Metadata API: +#' Metadata api is currently available to all users. +#' +#' @param base Airtable base ID +#' @param ... additional paramters +#' +#' @return list of schema +#' @export air_get_schema + +air_get_schema <- function(base,...){ + request_url <- sprintf("%s/%s/tables", air_meta_url, base) + request_url <- utils::URLencode(request_url) + + # call service: + res <- httr::GET( + request_url, + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ) + ) + + air_validate(res) + # may need a new air_parse function + + res_content <- httr::content(res,as = "text") + + schema <- jsonlite::fromJSON(res_content) + + return(schema) +} + +type_option_map <-function(){ + + "https://airtable.com/developers/web/api/field-model" + + # time options are deeply nested + + + +} + +air_fields_df_template <- function(name,description, type, options = NA){ + df <- data.frame(name = name, + description = description, + type = type, + options = options) + + return(df) +} + +#' Template for lists that describe tables in Airtable +#' +#' @param table_name String. Name of table +#' @param description String. Description of the table +#' @param fields_df Data frame. Data frame describing the field in a table. +#' Should contain a name, description,type, and options field. +#' if +#' +#' @return List with table name, description, and fields +#' @export air_table_template +#' +#' @examples +air_table_template <- function(table_name, description, fields_df ){ + + valid_cols <- c("description","name","type","options") + + invalid_names_check <- !names(fields_df)%in%valid_cols + + if(any(invalid_names_check)){ + + msg <- glue::glue("Invalid column name in field_df: {names(fields_df)[invalid_names_check]} + valid column names are: {valid_cols}") + stop(msg) + } + + # check for required columns + required_cols <- valid_cols[1:3] + + required_names_check <- !required_cols %in% names(fields_df) + + if(any(required_names_check)){ + + msg <- glue::glue("Missing required column name in fields_df: {required_cols[required_names_check]} + required column names are: {required_cols}") + stop(msg) + } + + ## check that types have necessary options - TODO + + ## check that options have appropriate values - TODO + + ## create output + table_list <- list( + "name" = table_name, + "description" = description, + "fields" = fields_df + ) + + return(table_list) +} + +#' A function to create new tables in a base +#' +#' Takes a list object with appropriate arguments (see \code{air_table_template}) +#' converts it to JSON then adds it to the specified base. +#' +#' @note See https://airtable.com/developers/web/api/create-table +#' @param base String. ID for the base +#' @param table_list List. see \code{air_table_template} +#' +#' @return Data frame of table schema +#' @export air_create_table +#' +#' @examples +air_create_table <- function(base, table_list){ + request_url <- sprintf("%s/%s/tables", air_meta_url, base) + request_url <- utils::URLencode(request_url) + + fields_json <- jsonlite::toJSON(table_list,pretty = TRUE,auto_unbox = TRUE) + + # call service: + res <- httr::POST( + request_url, + httr::content_type("application/json"), + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ), + body = fields_json + ) + + air_validate(res) + # may need a new air_parse function + + res_content <- httr::content(res,as = "text") + + schema <- jsonlite::fromJSON(res_content) + + return(schema) +} + + From b02589eccd1e81e4c836da1627d40ff67be67685 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Wed, 15 Feb 2023 12:29:19 -0600 Subject: [PATCH 067/126] added defaults for metadata fields_df --- R/air_dump.R | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index e036752..6068917 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -42,12 +42,16 @@ set_diff <- function(x,y){ #' @param base String. Base id #' @param meta_data Data frame. Contains metadata records. From air_generate_metadata* #' @param table_name String. name of the metadata table. default is "Meta Data" +#' @param description Character vector. Descriptions of metadata table fields +#' @param type Character vector. Column types for metadata table fields. see https://airtable.com/developers/web/api/field-model +#' @param options Data frame. Options for fields in metadata table. #' #' @return List with outcome from creating the table and inserting the records #' @export air_create_metadata_table #' #' @examples -air_create_metadata_table <- function(base,meta_data,table_name = "Meta Data"){ +air_create_metadata_table <- function(base,meta_data,table_name = "Meta Data", description = NA, + type = "singleLineText", options = NA){ # check for meta data table ## if exists, stop @@ -63,8 +67,7 @@ air_create_metadata_table <- function(base,meta_data,table_name = "Meta Data"){ # create fields_df # add description for standard names - description <- NA - type <- "singleLineText" + # if(setequal(names(meta_data), c("field_name", "table_name", "field_desc", @@ -87,8 +90,8 @@ air_create_metadata_table <- function(base,meta_data,table_name = "Meta Data"){ fields_df <- air_fields_df_template(name = names(meta_data), description = description, - type = "singleLineText", - options = NA) + type = type, + options = options) # create list describing table From b422473cf6fa3e7e1b4e77e6b0dd0df74ad444ce Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Wed, 1 Mar 2023 16:46:20 -0600 Subject: [PATCH 068/126] added descriptive metadata table --- R/air_dump.R | 278 +++++++++++++++++++++++++++++++++++++++---- R/air_metadata_api.R | 52 ++++++++ R/airtabler.R | 7 +- 3 files changed, 315 insertions(+), 22 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index 6068917..03c2b1c 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -42,7 +42,7 @@ set_diff <- function(x,y){ #' @param base String. Base id #' @param meta_data Data frame. Contains metadata records. From air_generate_metadata* #' @param table_name String. name of the metadata table. default is "Meta Data" -#' @param description Character vector. Descriptions of metadata table fields +#' @param field_descriptions Character vector. Descriptions of metadata table fields #' @param type Character vector. Column types for metadata table fields. see https://airtable.com/developers/web/api/field-model #' @param options Data frame. Options for fields in metadata table. #' @@ -50,12 +50,12 @@ set_diff <- function(x,y){ #' @export air_create_metadata_table #' #' @examples -air_create_metadata_table <- function(base,meta_data,table_name = "Meta Data", description = NA, +air_create_metadata_table <- function(base,meta_data,table_name = "Meta Data", field_descriptions = NA, type = "singleLineText", options = NA){ # check for meta data table ## if exists, stop - schema <- air_get_schema("appVjIfAo8AJlfTkx") + schema <- air_get_schema(base) if(table_name %in% schema$tables$name){ msg <- glue::glue("{table_name} already exists in the base {base}. @@ -75,21 +75,21 @@ air_create_metadata_table <- function(base,meta_data,table_name = "Meta Data", "field_opts", "primary_key"))){ # create description object - description <- c("https://schema.org/name", - "https://schema.org/name", - "https://schema.org/description", - "https://schema.org/category", - "https://schema.org/identifier", - "https://schema.org/identifier", - "https://schema.org/option", - "https://schema.org/Boolean" + field_descriptions <- c("https://schema.org/name", + "https://schema.org/name", + "https://schema.org/description", + "https://schema.org/category", + "https://schema.org/identifier", + "https://schema.org/identifier", + "https://schema.org/option", + "https://schema.org/Boolean" ) } fields_df <- air_fields_df_template(name = names(meta_data), - description = description, + description = field_descriptions, type = type, options = options) @@ -127,22 +127,239 @@ air_create_metadata_table <- function(base,meta_data,table_name = "Meta Data", # add the description table to the base -air_insert_description_table <- function(base,description, table_name = "Description"){ +#' Create the descriptive metadata table for the base +#' +#' @details DCMI terms can be found here \url{https://www.dublincore.org/specifications/dublin-core/dcmi-terms/} +#' +#' @param base String. Base id +#' @param description Data frame. Description from air_get_base_description* or air_generate_base_description +#' @param table_name String. Name of description table +#' @param field_descriptions Character vector. Descriptions of metadata table fields. If NA, DCMI terms will be used where possible. +#' @param type Character vector. Column types for metadata table fields. see \url{https://airtable.com/developers/web/api/field-model} +#' @param options Data frame. Options for fields in metadata table. +#' +#' +#' @return List. Outputs from creating the table and inserting the records +#' @export air_create_description_table +#' +#' @examples +#'\dontrun{ +#' base = "appVjIfAo8AJlfTkx" +#' table_name= "description" +#' +#' description <- air_generate_base_description(title = "Example Base", +#' creator = "Collin Schwantes") +#' +#' air_create_description_table(base,description,table_name) +#'} +#' +air_create_description_table <- function(base, + description, + table_name = "Description", + field_descriptions = NA, + type = "singleLineText", + options = NA){ + + # check for description table + ## if exists, stop + schema <- air_get_schema(base) + + if(table_name %in% schema$tables$name){ + msg <- glue::glue("{table_name} already exists in the base {base}. + Please use air_update_description_table to update the description table + or delete the table and re-run the function") + stop(msg) + } + + + # create fields_df + # add description for dcmi terms + # check for dcmi terms + if(all(is.na(field_descriptions))){ + # create a placeholder dataframe + description_terms_df <- data.frame(col_names = names(description), dcmi_term = NA) + + # setup sprintf pattern + dcmi_uri <- "http://purl.org/dc/terms/%s" + + # check for a match for each dcmi term + description_terms_df_2 <- purrr::map_dfr(deposits::dcmi_terms(), function(dcmi_term){ + + # get index position for a term + dcmi_term_pos <- stringr::str_which(description_terms_df$col_names,pattern =dcmi_term,negate = FALSE) + + if(!rlang::is_empty(dcmi_term_pos)){ + + description_terms_df[dcmi_term_pos,"dcmi_term"] <- sprintf(dcmi_uri,dcmi_term) + + return(description_terms_df[dcmi_term_pos,]) + } + + return(NULL) + }) + + # get undescribed terms + + description_terms_df_3 <- dplyr::anti_join(description_terms_df,description_terms_df_2,"col_names") + + description_terms_df_4 <- rbind(description_terms_df_2,description_terms_df_3) + + description_terms_df_5 <- description_terms_df_4[match(names(description),description_terms_df_4$col_names),] + + field_descriptions <- description_terms_df_5$dcmi_term + + } + + fields_df <- air_fields_df_template(name = names(description), + description = field_descriptions, + type = type, + options = options) + + # create list describing table + + table_list <- air_table_template(table_name = table_name, + description = "descriptive metadata for the base", + fields_df = fields_df) + # create table + + outcome_create_table <- air_create_table(base, table_list) + + # insert data + + outcome_insert_data <- tryCatch( + air_insert_data_frame(base = base,table_name = table_name,records = description), + error=function(cond) { + + warning(cond) + + return("data not inserted") + }) + + if(is.character(outcome_insert_data)){ + stop("Table created but data not inserted. Check field types then use + air_insert_data_frame or air_update_description_table to add metadata + records.") + } + + + return(list("create_table" = outcome_create_table, + "insert_data" = outcome_insert_data)) } # update metadata table -air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data"){ +#' Update the descriptive metadata table +#' +#' @param base String. Base id +#' @param meta_data Data frame. Contains metadata records. From air_generate_metadata* +#' @param table_name String. Name of metadata table +#' @param join_field String. Name of field to join new and current metadata. Likely \code{field_id} +#' @param record_id_field String. Name of record id field. Like \code{id} +#' +#' @return +#' @export +#' +#' @examples +#'\dontrun{ +#' base = "appVjIfAo8AJlfTkx" +#' metadata <- air_generate_metadata_from_api(base = base) +#' air_update_metadata_table(base,metadata) +#'} +#' +air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", join_field = "field_id", record_id_field = "id"){ + + schema <- air_get_schema(base) # check for Meta Data table ## if no meta data table, stop + + if(!table_name %in% schema$tables$name){ + msg <- glue::glue("No table called {table_name} in base {base}. + Please use air_create_metadata_table to create the metadata table + or create it manually.") + stop(msg) + } + + ## get table_id + + table_id <- schema$tables[schema$tables$name == table_name,"id"] + # pull down current meta data table + + current_metadata_table <- fetch_all(base,table_name) + + #create any new fields from meta_data + + update_log <- list( + fields_created = NA, + records_updated = NA, + records_inserted = NA, + records_deleted = NA + ) + + col_check <- !names(meta_data) %in% names(current_metadata_table) + + if(all(col_check)){ + msg <- glue::glue("The meta_data object and the metadata table in your base, {table_name}, share + no fields.") + warning(msg) + } + + if(any(col_check)){ + cols_to_create <- names(meta_data)[col_check] + + fields_created <- air_create_field(base = base, + table_id = table_id, + name = cols_to_create) + + message(fields_created) + + update_log$fields_created <- fields_created + + } + # compare with updated values + ## use field ids + + ## assumes a certain structure for metadata + + min_update_df <- current_metadata_table[,c(join_field,record_id_field)] + + records_to_update <- dplyr::inner_join(meta_data,min_update_df,by = join_field ) + # update records - # create new records + + records_updated <- air_update_data_frame(base, table_name, records_to_update$id,records_to_update) + + update_log$records_updated <- records_updated + + # insert new records + + records_to_insert <- dplyr::anti_join(meta_data,min_update_df,by = join_field) + + records_inserted <- air_insert_data_frame(base, table_name,records_to_insert) + + update_log$records_inserted <- records_inserted + + # drop records no longer in meta data + records_to_delete <- dplyr::anti_join(min_update_df,meta_data,by = join_field) + + + if(nrow(records_to_delete) >0){ + records_deleted <- purrr::map(records_to_delete$id, function(id){ + air_delete(base, table_name,id) + }) + } else { + records_deleted <- "No records deleted" + } + + update_log$records_deleted <- records_deleted + + + return(update_log) } # update description table @@ -285,7 +502,7 @@ air_generate_metadata_from_api <- function(base, }) -return(metadata_df) + return(metadata_df) } ##air_insert @@ -366,16 +583,20 @@ air_get_base_description_from_table<- function(base, table_name){ #' #' @param title String. Title is a property that refers to the name or names by #' which a resource is formally known. +#' @param creator String. Person or people who created the base #' @param primary_contact String. Person or entity primarily responsible for #' making the content of a resource #' @param email String. Email of primary_contact -#' @param base_description String. This property refers to the description of +#' @param description String. This property refers to the description of #' the content of a resource. The description is a potentially rich source of #' indexable terms and assist the users in their selection of an appropriate #' resource. +#' @param contributor String. An entity responsible for making contributions to the resource. +#' @param identifier String. An unambiguous reference to the resource within a given context. +#' @param license String. A legal document giving official permission to do something with the resource. "CC BY 4.0" #' @param ... String. Additional descriptive metadata elements. See details. #' Additional elements can be added as name pair values e.g. -#' \code{license = "CC BY 4.0", is_part_of = "https://doi.org/10.48321/MyDMP01"} +#' \code{ isPartOf = "https://doi.org/00.00000/MyPaper01", isReferencedBy = "https://doi.org/10.48321/MyDMP01"} #' #' @return data.frame with descriptive metadata #' @export @@ -392,8 +613,25 @@ air_get_base_description_from_table<- function(base, table_name){ #' is_part_of = "https://doi.org/10.5072/zenodo_sandbox.1062705" #' ) #' -air_generate_base_description <- function(title = NA,primary_contact= NA,email = NA, base_description = NA,...){ - desc_table <- data.frame(title,primary_contact,email,base_description,...) +air_generate_base_description <- function(title = NA, + creator= NA, + created=NA, + primary_contact=NA, + email = NA, + description = NA, + contributor = NA, + identifier =NA, + license = NA,...){ + desc_table <- data.frame(title = title, + creator= creator, + created=created, + primary_contact=primary_contact, + email = email, + description = description, + contributor = contributor, + identifier =identifier, + license = license, + ...) return(desc_table) } diff --git a/R/air_metadata_api.R b/R/air_metadata_api.R index 27484ba..e33aac9 100644 --- a/R/air_metadata_api.R +++ b/R/air_metadata_api.R @@ -143,4 +143,56 @@ air_create_table <- function(base, table_list){ return(schema) } +#' Create a new field in a table +#' +#' See https://airtable.com/developers/web/api/create-field +#' +#' @param base String. Base id +#' @param table_id String. Table id. Can be found using \code{air_get_schema} +#' @param name String. Name of the field +#' @param description String. Description of the field +#' @param type String. Type of field. See https://airtable.com/developers/web/api/field-model +#' @param options Data frame. See https://airtable.com/developers/web/api/field-model +#' +#' @return description of newly created field as a list +#' @export air_create_field +#' +#' @examples +air_create_field <- function(base, + table_id, + name, + description = NA, + type = "singleLineText", + options= NA){ + + field_df <- air_fields_df_template(name = name,description = description, type = type, options = options) + + "https://api.airtable.com/v0/meta/bases/{baseId}/tables/{tableId}/fields" + + request_url <- sprintf("%s/%s/tables/%s/fields", air_meta_url, base,table_id) + request_url <- utils::URLencode(request_url) + + fields_json <- jsonlite::toJSON(field_df,pretty = TRUE,auto_unbox = TRUE) + + # call service: + res <- httr::POST( + request_url, + httr::content_type("application/json"), + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ), + body = fields_json + ) + + air_validate(res) + # may need a new air_parse function + + res_content <- httr::content(res,as = "text") + + schema <- jsonlite::fromJSON(res_content) + + return(schema) + +} + diff --git a/R/airtabler.R b/R/airtabler.R index 4b35170..0ef3b66 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -53,14 +53,17 @@ air_secret_key <- function(){ #' Get a list of records or retrieve a single #' -#' You can retrieve records in an order of a view by providing the name or ID of +#' Retrieve records or a single record from a table. If you provide a record_id, +#' you cannot specify fields, views, or filterFormulas. +#' +#'You can retrieve records in an order of a view by providing the name or ID of #' the view in the view query parameter. The results will include only records #' visible in the order they are displayed. #' #' @param base Airtable base #' @param table_name Table name #' @param record_id (optional) Use record ID argument to retrieve an existing -#' record details +#' record details. See \url{https://airtable.com/developers/web/api/get-record} #' @param limit (optional) A limit on the number of records to be returned. #' Limit can range between 1 and 100. #' @param offset (optional) Page offset returned by the previous list-records From e0c908999fb365aaade4fa583ca40eb490bc52da Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Wed, 1 Mar 2023 17:06:32 -0600 Subject: [PATCH 069/126] added a parse all sheets option to get_attachments --- R/air_get_attachments.R | 5 +++-- R/read_excel_url.R | 22 ++++++++++++++++++---- man/air_get_attachments.Rd | 3 +++ man/read_excel_url.Rd | 6 ++++-- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/R/air_get_attachments.R b/R/air_get_attachments.R index e0c8617..1003ef4 100644 --- a/R/air_get_attachments.R +++ b/R/air_get_attachments.R @@ -13,12 +13,13 @@ #' @param dir_name String. Where should files be downloaded to? #' Will create the folder if it does not exist. #' @param skip Numeric. How many lines should be skipped? See \code{readxl::read_excel} skip. +#' @param parse_all_sheets Logical. Should all sheets in spreadsheet be parsed? #' #' @return named list of data frames #' @export air_get_attachments #' #' @examples -air_get_attachments <- function(base, table_name, field, download_file = FALSE, dir_name = "downloads", extract_type ="excel", extract_field ="excel_extract", skip = 0, ...){ +air_get_attachments <- function(base, table_name, field, download_file = FALSE, dir_name = "downloads", extract_type ="excel", extract_field ="excel_extract", skip = 0, parse_all_sheets = FALSE, ...){ #browser() # get data x <- fetch_all(base,table_name,...) @@ -44,7 +45,7 @@ air_get_attachments <- function(base, table_name, field, download_file = FALSE, warning(sprintf("Record ID %s is null",ID)) return(NULL) } - read_excel_url(x$url, skip = skip) ## need to be able to pass additional arguments + read_excel_url(x$url, skip = skip,parse_all_sheets = parse_all_sheets) ## need to be able to pass additional arguments }) ## add extract to data frame ---- diff --git a/R/read_excel_url.R b/R/read_excel_url.R index c0ea6dc..7ba2740 100644 --- a/R/read_excel_url.R +++ b/R/read_excel_url.R @@ -1,16 +1,30 @@ #' Read an excel file from URL #' #' Extends \code{readxl::read_excel} to allow for reading from a URL. +#' #' @param url String. Url for file #' @param fileext String. File extension for temp file #' @param ... additional arguments to pass to \code{read_excel} +#' @param parse_all_sheets Logical. Should all sheets be parsed? #' -#' @return tibble +#' @return tibble or list of tibbles if parse_all_sheets = TRUE #' @export read_excel_url #' -#' @examples -read_excel_url <- function(url, fileext= ".xslx",...){ +read_excel_url <- function(url, fileext= ".xslx",parse_all_sheets = FALSE,...){ tmp <- tempfile(fileext = ".xslx") curl::curl_download(url, tmp ) - readxl::read_excel(tmp,...) + if(parse_all_sheets){ + sheets <- readxl::excel_sheets(tmp) + xl_list <- purrr::map(sheets,function(x){ + readxl::read_excel(path = tmp,sheet = x,...) + }) + + names(xl_list) <- sheets + + return(xl_list) + } + + readxl::read_excel(path = tmp,...) + + } diff --git a/man/air_get_attachments.Rd b/man/air_get_attachments.Rd index fba4398..3e6db31 100644 --- a/man/air_get_attachments.Rd +++ b/man/air_get_attachments.Rd @@ -13,6 +13,7 @@ air_get_attachments( extract_type = "excel", extract_field = "excel_extract", skip = 0, + parse_all_sheets = FALSE, ... ) } @@ -35,6 +36,8 @@ Should be one of: excel} \item{skip}{Numeric. How many lines should be skipped? See \code{readxl::read_excel} skip.} +\item{parse_all_sheets}{Logical. Should all sheets in spreadsheet be parsed?} + \item{...}{Additional arguments to pass to \code{air_get}} } \value{ diff --git a/man/read_excel_url.Rd b/man/read_excel_url.Rd index d41a521..55c25e9 100644 --- a/man/read_excel_url.Rd +++ b/man/read_excel_url.Rd @@ -4,17 +4,19 @@ \alias{read_excel_url} \title{Read an excel file from URL} \usage{ -read_excel_url(url, fileext = ".xslx", ...) +read_excel_url(url, fileext = ".xslx", parse_all_sheets = FALSE, ...) } \arguments{ \item{url}{String. Url for file} \item{fileext}{String. File extension for temp file} +\item{parse_all_sheets}{Logical. Should all sheets be parsed?} + \item{...}{additional arguments to pass to \code{read_excel}} } \value{ -tibble +tibble or list of tibbles if parse_all_sheets = TRUE } \description{ Extends \code{readxl::read_excel} to allow for reading from a URL. From 17575027d488131b44cf00e5b9ddf228e8e3ffd3 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 31 Mar 2023 12:30:38 -0600 Subject: [PATCH 070/126] adding testing script and .env to git ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index bfd2ad0..8e10a1d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ .Rhistory .RData readme.html +testing_script.R +.env From 5f81a4467fce29c721f97532229a7a61867ca4fe Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 4 Apr 2023 10:58:42 -0600 Subject: [PATCH 071/126] adding vignette and making templates more flexible --- .gitignore | 1 + DESCRIPTION | 4 + R/air_dump.R | 9 +- R/air_metadata_api.R | 50 ++++++- vignettes/.gitignore | 2 + vignettes/Generate-Metadata-and-Backup.Rmd | 144 +++++++++++++++++++++ 6 files changed, 200 insertions(+), 10 deletions(-) create mode 100644 vignettes/.gitignore create mode 100644 vignettes/Generate-Metadata-and-Backup.Rmd diff --git a/.gitignore b/.gitignore index 8e10a1d..dde8cc7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ readme.html testing_script.R .env +inst/doc diff --git a/DESCRIPTION b/DESCRIPTION index 45d6fe7..575601e 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -26,3 +26,7 @@ Imports: rlang, stringr RoxygenNote: 7.2.1 +Suggests: + knitr, + rmarkdown +VignetteBuilder: knitr diff --git a/R/air_dump.R b/R/air_dump.R index 03c2b1c..b070c44 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -249,7 +249,7 @@ air_create_description_table <- function(base, # update metadata table -#' Update the descriptive metadata table +#' Update the structural metadata table #' #' @param base String. Base id #' @param meta_data Data frame. Contains metadata records. From air_generate_metadata* @@ -364,8 +364,9 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j # update description table air_update_description_table <- function(base,description, table_name = "Description"){ - # check for description + current_description_table <- air_fetch(base,table_name) + # update records } @@ -562,7 +563,7 @@ air_get_base_description_from_table<- function(base, table_name){ # to snake case names(desc_table) <- snakecase::to_snake_case(names(desc_table)) - required_fields <- c("title","primary_contact","email","base_description") + required_fields <- c("title","primary_contact","email","description") if(all(required_fields %in% names(desc_table))){ return(desc_table) } else { @@ -622,7 +623,7 @@ air_generate_base_description <- function(title = NA, contributor = NA, identifier =NA, license = NA,...){ - desc_table <- data.frame(title = title, + desc_table <- tibble::tibble(title = title, creator= creator, created=created, primary_contact=primary_contact, diff --git a/R/air_metadata_api.R b/R/air_metadata_api.R index e33aac9..237e6a8 100644 --- a/R/air_metadata_api.R +++ b/R/air_metadata_api.R @@ -44,15 +44,49 @@ type_option_map <-function(){ } +#' Template for for creating a table from a dataframe +#' +#' @param name +#' @param description +#' @param type +#' @param options +#' +#' @return +#' @export +#' +#' @examples +#' air_fields_df_template <- function(name,description, type, options = NA){ - df <- data.frame(name = name, - description = description, - type = type, - options = options) + df <- tibble::tibble(name = name, + description = description, + type = type, + options = options) return(df) } +air_fields_list_from_template <- function(df){ + ## create a list of field objects + purrr::pmap(df, function(name, + type, + description, + options){ + + field_list <- list(name = name, + type = type) + + if(!is.na(description)){ + field_list$description <- description + } + + if(!all(is.na(options))){ + field_list$options <- options + } + return(field_list) + }) + +} + #' Template for lists that describe tables in Airtable #' #' @param table_name String. Name of table @@ -94,14 +128,18 @@ air_table_template <- function(table_name, description, fields_df ){ ## check that options have appropriate values - TODO + ## convert data frame to list of field objects for easier translation to JSON + + fields_list <- air_fields_list_from_template(df = fields_df) + ## create output table_list <- list( "name" = table_name, "description" = description, - "fields" = fields_df + "fields" = fields_list ) - return(table_list) + return(table_list) } #' A function to create new tables in a base diff --git a/vignettes/.gitignore b/vignettes/.gitignore new file mode 100644 index 0000000..097b241 --- /dev/null +++ b/vignettes/.gitignore @@ -0,0 +1,2 @@ +*.html +*.R diff --git a/vignettes/Generate-Metadata-and-Backup.Rmd b/vignettes/Generate-Metadata-and-Backup.Rmd new file mode 100644 index 0000000..a324c58 --- /dev/null +++ b/vignettes/Generate-Metadata-and-Backup.Rmd @@ -0,0 +1,144 @@ +--- +title: "Generate-Metadata-and-Backup" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Generate-Metadata-and-Backup} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(airtabler) +``` + +This vignette will demonstrate how to generate and update metadata for an Airtable +base using the [Airtable metadata api](https://airtable.com/developers/web/api/get-base-schema) then export the base to create an offline backup. + +*Note* Make sure that you have the [environment variable](https://github.com/gaborcsardi/dotenv) `AIRTABLE_API_KEY` set with your [Airtable API token](https://airtable.com/create/tokens). Make sure your token has schema read and write priveleges. + +## Creating metadata from schema + +`air_generate_metadata_from_api` creates a data frame that holds the structural metadata for your base. Structural metadata describes how entities relate to one another, in this case, how fields relate to tables within the base. + + +```{r generate metadata} +# set your base id +base = "appVjIfAo8AJlfTkx" +## create a list +api_metadata <- air_generate_metadata_from_api(base = "appVjIfAo8AJlfTkx") +str(api_metadata) +``` + +## Adding a metadata table to the base + +```{r add metadata table} +status <- air_create_metadata_table(base = base,meta_data = api_metadata) +``` + +## Create a descriptive metadata table for the base + +The `description` table will describe who created the base, when, and why. +Descriptive metadata is important for long term storage and reuse. The default +terms in the `description` table match core DCMI terms. Additional terms can be +added. + +```{r descriptive metadata} +## create a basic descriptive metadata table + +description <- air_generate_base_description(title = "Base Title", + creator = "Collin Schwantes", + created = "2023-04-03", + description = "This base is an example for airtabler") + + +# We could have provided keywords as comma separated values but lets make things +# more interesting by presenting them as a vector +description_with_keywords <- air_generate_base_description(title = "Base Title", + creator = "Collin Schwantes", + created = "2023-04-03", + description = "This base is an example for airtabler", + keywords = list(c("Example","R","Package","Airtable"))) + +``` + +## Add a description table to the base + +```{r add descriptive metadata} + +## add our vanilla description table - preferred way to work with description data +air_create_description_table(base = base,description = description) + + +## add our description with keywords and - we want keywords to be multiple select +## Note this is not the preferred method but it is possible + +length(names(description_with_keywords)) + +create_choices <- function(x){ + + choice_list <- list() + + choice_list$choices <- purrr::map(x,function(x_item){ + list(name = x_item) + }) + + return(list(choice_list)) # wrap in an extra list to +} + +keyword_options <- create_choices(description_with_keywords$keywords[[1]]) + +air_create_description_table(base = base,description = description_with_keywords, + type = c(rep("singleLineText",9),"multipleSelects"), + options = c(rep(NA,9),keyword_options)) + + +``` + +## Updating metadata tables + +Your data will change so your metadata should update as well. + +### Structural Metadata +```{r update structural metadata} + +# get your metadata from the api + metadata <- air_generate_metadata_from_api(base = base) + +# run the function +update_log <- air_update_metadata_table(base,metadata) + +# the update log provides an overview of records that were updated, inserted, or deleted +# and fields that were created in the event that the structure of your metadata table +# changed. + +``` + +### Descriptive Metadata + + + +```{r update descriptive metadata} + +base_description <- air_get_base_description_from_table(base = base,table_name = "Description") + +base_description$description <- "updating the description to be more descriptive" + +air_update_description_table(base = base,description = base_description,table_name = "description") + +``` + + +## Using the metadata to create a backup + + + + + + From 8a199b33a7a59401250a58a60d638b1685fb71db Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 4 Apr 2023 16:42:14 -0600 Subject: [PATCH 072/126] attachment fields may be empty so a warning is more appropriate --- R/air_download_attachments.R | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/R/air_download_attachments.R b/R/air_download_attachments.R index f2710ed..8082144 100644 --- a/R/air_download_attachments.R +++ b/R/air_download_attachments.R @@ -31,7 +31,16 @@ air_download_attachments <- function(x, field, dir_name = "downloads",...){ if(!is.list(x[,field])){ error_msg <- glue::glue("{field} is not of class list. Verify the name of the column used to store attachments in airtable") - rlang::abort(error_msg) + rlang::warn(error_msg) + + field_file_paths <- sprintf("%s_file_paths",field) + + x$file_path <- NA + + # using dynamic names in case a base has multiple file attachment + # columns + x <- dplyr::rename(x,{{field_file_paths}} := file_path) + return(x) } ### subset to necessary records ---- From 1b45239e2a81acacfe34a377d051bf3379f9341e Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 4 Apr 2023 16:43:21 -0600 Subject: [PATCH 073/126] added update description function and fixed a name issue --- R/air_dump.R | 134 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 127 insertions(+), 7 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index b070c44..3c4b8a2 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -363,12 +363,130 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j } # update description table -air_update_description_table <- function(base,description, table_name = "Description"){ +#' Update the description table +#' +#' Update the descriptive metadata table in airtable +#' +#' @param base String. Base id +#' @param description Data frame. Contains updated description +#' @param table_name String. Name of description table +#' @param join_field String. Field to perform join on +#' @param record_id_field String. Name of the record id field +#' +#' @return list that logs updates +#' @export +#' +#' @examples +air_update_description_table <- function(base,description, table_name = "Description", join_field = "title", record_id_field = "id"){ + + # check for description - current_description_table <- air_fetch(base,table_name) + schema <- air_get_schema(base) + + # check for Description table + ## if no table, stop + + if(!table_name %in% schema$tables$name){ + msg <- glue::glue("No table called {table_name} in base {base}. + Please use air_create_description_table to create the descriptive + metadata table or create it manually.") + stop(msg) + } + + ## create empty update log list + + update_log <- list( + fields_created = NA, + records_updated = NA, + records_inserted = NA, + records_deleted = NA + ) + + + ## get table_id + + table_id <- schema$tables[schema$tables$name == table_name,"id"] + table_pos <- which(schema$tables$id == table_id) + # get column names from schema because empty columns are pulled + table_columns <- schema$tables$fields[[table_pos]]$name + ## check columns + col_check <- !names(description) %in% table_columns + + if(all(col_check)){ + msg <- glue::glue("The description object and the metadata table in your base, {table_name}, share + no fields.") + warning(msg) + } + #create any new fields from description + if(any(col_check)){ + cols_to_create <- names(description)[col_check] + + fields_created <- air_create_field(base = base, + table_id = table_id, + name = cols_to_create) + + message(fields_created) + + update_log$fields_created <- fields_created + + } + + # pull down current table + + current_metadata_table <- fetch_all(base,table_name) + # convert to tibble for more consistent behavior in joins + current_metadata_table<- tibble::as_tibble(current_metadata_table) + current_metadata_table <- current_metadata_table |> + dplyr::select(-createdTime) + + # compare with updated values + ## use field ids + ## assumes a certain structure for metadata + min_fields <- unique(join_field,record_id_field) + + min_update_df <- current_metadata_table[,min_fields] + + records_to_update <- dplyr::inner_join(description,min_update_df,by = join_field ) # update records + records_updated <- air_update_data_frame(base, table_name, records_to_update$id,records_to_update) + + update_log$records_updated <- records_updated + + # insert new records -- each table describes a single base so there should be + # no records to insert + + records_to_insert <- dplyr::anti_join(description,min_update_df,by = join_field) + + if(nrow(records_to_insert) > 0){ + warning("Inserting a new record into the base. Check that the value in {join_field} matches + between current and updated description") + } + + records_inserted <- air_insert_data_frame(base, table_name,records_to_insert) + + update_log$records_inserted <- records_inserted + + + # drop records no longer in meta data + + records_to_delete <- dplyr::anti_join(min_update_df,description,by = join_field) + + + if(nrow(records_to_delete) >0){ + records_deleted <- purrr::map(records_to_delete$id, function(id){ + air_delete(base, table_name,id) + }) + } else { + records_deleted <- "No records deleted" + } + + update_log$records_deleted <- records_deleted + + + return(update_log) + } @@ -415,7 +533,7 @@ air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, f tables$field_desc <- "unique id assigned by airtable" tables$field_type <- "singleLineText" tables$field_id <- NA - tables$field_opt_name <- NA + tables$field_opts <- NA str_metadata <- rbind(str_metadata,tables) @@ -557,11 +675,13 @@ air_generate_metadata <- function(base, table_names,limit=1){ #' @return data.frame with descriptive metadata. #' @export -air_get_base_description_from_table<- function(base, table_name){ +air_get_base_description_from_table<- function(base, table_name,field_names_to_snakecase = TRUE){ #fetch table desc_table <- airtabler::fetch_all(base,table_name) # to snake case + if(field_names_to_snakecase){ names(desc_table) <- snakecase::to_snake_case(names(desc_table)) + } required_fields <- c("title","primary_contact","email","description") if(all(required_fields %in% names(desc_table))){ @@ -570,7 +690,7 @@ air_get_base_description_from_table<- function(base, table_name){ missing_rf <- required_fields[!required_fields %in% names(desc_table)] - desc_table[missing_rf] <- NA + desc_table[missing_rf] <- "" return(desc_table) } @@ -893,7 +1013,7 @@ flatten_col_to_chr <- function(data_frame){ #' @param output_dir String. Folder containing output files #' @param overwrite Logical. Should outputs be overwritten if they already exist? #' -#' @return list. Returns the table_list object +#' @return Vector of file paths #' @export air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE){ @@ -940,7 +1060,7 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) file.copy(from = outputs_list,to = output_dir_path_final,recursive = FALSE ,copy.mode = TRUE) - invisible(table_list) + return(list.files(output_dir_path_final,full.names = TRUE)) } From 47b1cf7a2ef76f02f8706e2bf0e384d88c672323 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 4 Apr 2023 16:44:13 -0600 Subject: [PATCH 074/126] make create_field capable of handling multiple fields --- R/air_metadata_api.R | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/R/air_metadata_api.R b/R/air_metadata_api.R index 237e6a8..0748109 100644 --- a/R/air_metadata_api.R +++ b/R/air_metadata_api.R @@ -205,31 +205,39 @@ air_create_field <- function(base, field_df <- air_fields_df_template(name = name,description = description, type = type, options = options) + + fields_list <- air_fields_list_from_template(field_df) + "https://api.airtable.com/v0/meta/bases/{baseId}/tables/{tableId}/fields" request_url <- sprintf("%s/%s/tables/%s/fields", air_meta_url, base,table_id) request_url <- utils::URLencode(request_url) - fields_json <- jsonlite::toJSON(field_df,pretty = TRUE,auto_unbox = TRUE) + ## fields must be created one at a time + schema_list <- purrr::map(fields_list,function(field_item){ + fields_json <- jsonlite::toJSON(field_item,pretty = TRUE,auto_unbox = TRUE) + + # call service: + res <- httr::POST( + request_url, + httr::content_type("application/json"), + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ), + body = fields_json + ) - # call service: - res <- httr::POST( - request_url, - httr::content_type("application/json"), - httr::add_headers( - Authorization = paste("Bearer", air_api_key()) - ), - body = fields_json - ) + air_validate(res) + # may need a new air_parse function - air_validate(res) - # may need a new air_parse function + res_content <- httr::content(res,as = "text") - res_content <- httr::content(res,as = "text") + schema <- jsonlite::fromJSON(res_content) - schema <- jsonlite::fromJSON(res_content) + return(schema) + }) - return(schema) + return(schema_list) } From 10270b60760e72499531d59e8ba68dff76dfb67e Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 4 Apr 2023 16:44:32 -0600 Subject: [PATCH 075/126] added vignette for generating metadata and backups --- vignettes/Generate-Metadata-and-Backup.Rmd | 59 ++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/vignettes/Generate-Metadata-and-Backup.Rmd b/vignettes/Generate-Metadata-and-Backup.Rmd index a324c58..c293c46 100644 --- a/vignettes/Generate-Metadata-and-Backup.Rmd +++ b/vignettes/Generate-Metadata-and-Backup.Rmd @@ -126,19 +126,72 @@ update_log <- air_update_metadata_table(base,metadata) ```{r update descriptive metadata} -base_description <- air_get_base_description_from_table(base = base,table_name = "Description") +base_description <- air_get_base_description_from_table(base = base,table_name = "Description", + field_names_to_snakecase = FALSE) -base_description$description <- "updating the description to be more descriptive" +base_description$description <- "Keep on updating" -air_update_description_table(base = base,description = base_description,table_name = "description") +## since the field types are already established, its a little easier to add multipleSelect keywords +base_description$keywords[[1]] <- append("New Keyword",base_description$keywords[[1]]) + +base_description <- base_description |> + dplyr::select(-createdTime) + +## if your base_description obj has a record id field, use that for the join. Default is title +air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id") ``` ## Using the metadata to create a backup +Now that you have your metadata tables setup, lets see how we can use them to +create backups. The workflow pulls data down from airtable into R where it can +be written to whatever format you like. There is a built in function for writing +to versioned to CSVs. This is the recommended workflow for creating backups. + +### Pull all data into R + +```{r dump to R} + +## pull down the metadata table from airtable +metadata <- air_get_metadata_from_table(base = base,table_name = "Meta Data",add_id_field = FALSE) + +airtable_base <- air_dump(base = base,metadata = metadata,description = base_description) + +summary(airtable_base) + +``` +### Create a set of versioned CSVs +This function creates a folder with a unique ID based on a hash of the list +passed to table_list. If the data in your base do not change, then the hash +won't change and no new version of the data will be written. +```{r dump to csv} + +# dump to csv +air_dump_to_csv(table_list = airtable_base) + +# Make a change to the description table to show how hashing works +base_description <- air_get_base_description_from_table(base = base,table_name = "Description", + field_names_to_snakecase = FALSE) + +## since the field types are already established, its a little easier to add multipleSelect keywords +base_description$keywords[[1]] <- append("How will the hash change?",base_description$keywords[[1]]) + +base_description <- base_description |> + dplyr::select(-createdTime) + +## if your base_description obj has a record id field, use that for the join. Default is title +air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id") + +## pull the changed based down +airtable_base_updated <- air_dump(base = base,metadata = metadata,description = base_description) + +## dump to csv +air_dump_to_csv(table_list = airtable_base_updated) +``` From 7dd6b4fe5e0a2d181be0ebb74ffd55bcb3fc3866 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Wed, 5 Apr 2023 14:54:07 -0600 Subject: [PATCH 076/126] added functions and example for bulk workspace download --- R/air_metadata_api.R | 31 +++++++++++- vignettes/Generate-Metadata-and-Backup.Rmd | 56 ++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/R/air_metadata_api.R b/R/air_metadata_api.R index 0748109..079cc9b 100644 --- a/R/air_metadata_api.R +++ b/R/air_metadata_api.R @@ -39,7 +39,7 @@ type_option_map <-function(){ "https://airtable.com/developers/web/api/field-model" # time options are deeply nested - + NULL } @@ -241,4 +241,33 @@ air_create_field <- function(base, } +#' Get list of bases for an Token +#' +#' @return list +#' @export +#' +#' @examples +air_list_bases <- function(request_url = "https://api.airtable.com/v0/meta/bases"){ + + request_url <- utils::URLencode(request_url) + + + # call service: + res <- httr::GET( + request_url, + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ) + ) + + air_validate(res) + # may need a new air_parse function + + res_content <- httr::content(res,as = "text") + + base_list <- jsonlite::fromJSON(res_content) + + return(base_list) +} + diff --git a/vignettes/Generate-Metadata-and-Backup.Rmd b/vignettes/Generate-Metadata-and-Backup.Rmd index c293c46..f14e89d 100644 --- a/vignettes/Generate-Metadata-and-Backup.Rmd +++ b/vignettes/Generate-Metadata-and-Backup.Rmd @@ -194,4 +194,60 @@ airtable_base_updated <- air_dump(base = base,metadata = metadata,description = air_dump_to_csv(table_list = airtable_base_updated) ``` +## Backup all bases in a workspace +*Note* This routine requires personal access tokens that can read and write +data and schema to all bases in a workspace. + +Here we are going to take advantage of the [`list bases`](https://airtable.com/developers/web/api/list-bases) +endpoint in the airtable API. It will list all bases that a token has access to. + +```{r workspace backup} + +# get all bases associated with token +bases <- air_list_bases() + +# generate the description for our second +description_2 <- air_generate_base_description(title = bases$bases$name[[2]],creator = "Collin Schwantes",created = Sys.Date(),description = "A base to demo bulk backups") + +description_log <- air_create_description_table(bases$bases$id[[2]],description = description_2) + +metadata_2 <- air_generate_metadata_from_api(bases$bases$id[[2]]) + +metadata_log <- air_create_metadata_table(bases$bases$id[[2]],meta_data = metadata_2) + +## add metadata and descriptive data to bases list + +base_descriptions <- purrr::map(bases$bases$id,function(base){ + air_get_base_description_from_table(base,table_name = "Description") +}) + + +base_metadata <- purrr::map(bases$bases$id,function(base){ + air_get_metadata_from_table(base,table_name = "Meta Data") +}) + + +bases$bases$description <- base_descriptions +bases$bases$metadata <- base_metadata + +base_df <- bases$bases + +for(i in 1:nrow(base_df)){ + base_item <- base_df[i,] + table_list <- air_dump(base = base_item$id ,metadata = base_item$metadata[[1]],description = base_item$description[[1]]) + + output_dir <- sprintf("Airtabler Workspace/%s", base_item$name) + air_dump_to_csv(table_list,output_dir = output_dir) +} + + +``` + +## CSVs to Frictionless + +```{r csv dump to frictionless} + +``` + +## Frictionless to deposits From e20d150b7d834b30f48f94b08380cd3996ae03f5 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 6 Apr 2023 08:48:04 -0600 Subject: [PATCH 077/126] added bulk dump to csv and eval False --- vignettes/Generate-Metadata-and-Backup.Rmd | 32 ++++++++-------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/vignettes/Generate-Metadata-and-Backup.Rmd b/vignettes/Generate-Metadata-and-Backup.Rmd index f14e89d..672eafb 100644 --- a/vignettes/Generate-Metadata-and-Backup.Rmd +++ b/vignettes/Generate-Metadata-and-Backup.Rmd @@ -28,7 +28,7 @@ base using the [Airtable metadata api](https://airtable.com/developers/web/api/g `air_generate_metadata_from_api` creates a data frame that holds the structural metadata for your base. Structural metadata describes how entities relate to one another, in this case, how fields relate to tables within the base. -```{r generate metadata} +```{r generate metadata, eval=FALSE} # set your base id base = "appVjIfAo8AJlfTkx" ## create a list @@ -38,7 +38,7 @@ str(api_metadata) ## Adding a metadata table to the base -```{r add metadata table} +```{r add metadata table, eval=FALSE} status <- air_create_metadata_table(base = base,meta_data = api_metadata) ``` @@ -49,11 +49,11 @@ Descriptive metadata is important for long term storage and reuse. The default terms in the `description` table match core DCMI terms. Additional terms can be added. -```{r descriptive metadata} +```{r descriptive metadata, eval=FALSE} ## create a basic descriptive metadata table description <- air_generate_base_description(title = "Base Title", - creator = "Collin Schwantes", + creator = "Arkady Darell", created = "2023-04-03", description = "This base is an example for airtabler") @@ -61,7 +61,7 @@ description <- air_generate_base_description(title = "Base Title", # We could have provided keywords as comma separated values but lets make things # more interesting by presenting them as a vector description_with_keywords <- air_generate_base_description(title = "Base Title", - creator = "Collin Schwantes", + creator = "Arkady Darell", created = "2023-04-03", description = "This base is an example for airtabler", keywords = list(c("Example","R","Package","Airtable"))) @@ -70,7 +70,7 @@ description_with_keywords <- air_generate_base_description(title = "Base Title", ## Add a description table to the base -```{r add descriptive metadata} +```{r add descriptive metadata, eval=FALSE} ## add our vanilla description table - preferred way to work with description data air_create_description_table(base = base,description = description) @@ -106,7 +106,7 @@ air_create_description_table(base = base,description = description_with_keywords Your data will change so your metadata should update as well. ### Structural Metadata -```{r update structural metadata} +```{r update structural metadata, eval=FALSE} # get your metadata from the api metadata <- air_generate_metadata_from_api(base = base) @@ -124,7 +124,7 @@ update_log <- air_update_metadata_table(base,metadata) -```{r update descriptive metadata} +```{r update descriptive metadata, eval=FALSE} base_description <- air_get_base_description_from_table(base = base,table_name = "Description", field_names_to_snakecase = FALSE) @@ -152,7 +152,7 @@ to versioned to CSVs. This is the recommended workflow for creating backups. ### Pull all data into R -```{r dump to R} +```{r dump to R, eval=FALSE} ## pull down the metadata table from airtable metadata <- air_get_metadata_from_table(base = base,table_name = "Meta Data",add_id_field = FALSE) @@ -169,7 +169,7 @@ This function creates a folder with a unique ID based on a hash of the list passed to table_list. If the data in your base do not change, then the hash won't change and no new version of the data will be written. -```{r dump to csv} +```{r dump to csv, eval=FALSE} # dump to csv air_dump_to_csv(table_list = airtable_base) @@ -202,13 +202,13 @@ data and schema to all bases in a workspace. Here we are going to take advantage of the [`list bases`](https://airtable.com/developers/web/api/list-bases) endpoint in the airtable API. It will list all bases that a token has access to. -```{r workspace backup} +```{r workspace backup, eval=FALSE} # get all bases associated with token bases <- air_list_bases() # generate the description for our second -description_2 <- air_generate_base_description(title = bases$bases$name[[2]],creator = "Collin Schwantes",created = Sys.Date(),description = "A base to demo bulk backups") +description_2 <- air_generate_base_description(title = bases$bases$name[[2]],creator = "Arkady Darell",created = Sys.Date(),description = "A base to demo bulk backups") description_log <- air_create_description_table(bases$bases$id[[2]],description = description_2) @@ -243,11 +243,3 @@ for(i in 1:nrow(base_df)){ ``` - -## CSVs to Frictionless - -```{r csv dump to frictionless} - -``` - -## Frictionless to deposits From e33eb38cb6f263a2e13a730fd4137ebd8ba5f6aa Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 6 Apr 2023 08:59:30 -0600 Subject: [PATCH 078/126] updated docs --- man/air_create_description_table.Rd | 49 ++++++++++++++++++++++ man/air_create_field.Rd | 34 +++++++++++++++ man/air_create_metadata_table.Rd | 34 +++++++++++++++ man/air_create_table.Rd | 23 ++++++++++ man/air_dump_to_csv.Rd | 2 +- man/air_fields_df_template.Rd | 14 +++++++ man/air_generate_base_description.Rd | 19 +++++++-- man/air_get.Rd | 6 ++- man/air_get_base_description_from_table.Rd | 6 ++- man/air_get_enterprise.Rd | 23 ++++++++++ man/air_get_schema.Rd | 12 +++++- man/air_list_bases.Rd | 14 +++++++ man/air_table_template.Rd | 23 ++++++++++ man/air_update_description_table.Rd | 31 ++++++++++++++ man/air_update_metadata_table.Rd | 36 ++++++++++++++++ 15 files changed, 319 insertions(+), 7 deletions(-) create mode 100644 man/air_create_description_table.Rd create mode 100644 man/air_create_field.Rd create mode 100644 man/air_create_metadata_table.Rd create mode 100644 man/air_create_table.Rd create mode 100644 man/air_fields_df_template.Rd create mode 100644 man/air_get_enterprise.Rd create mode 100644 man/air_list_bases.Rd create mode 100644 man/air_table_template.Rd create mode 100644 man/air_update_description_table.Rd create mode 100644 man/air_update_metadata_table.Rd diff --git a/man/air_create_description_table.Rd b/man/air_create_description_table.Rd new file mode 100644 index 0000000..d194617 --- /dev/null +++ b/man/air_create_description_table.Rd @@ -0,0 +1,49 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_create_description_table} +\alias{air_create_description_table} +\title{Create the descriptive metadata table for the base} +\usage{ +air_create_description_table( + base, + description, + table_name = "Description", + field_descriptions = NA, + type = "singleLineText", + options = NA +) +} +\arguments{ +\item{base}{String. Base id} + +\item{description}{Data frame. Description from air_get_base_description* or air_generate_base_description} + +\item{table_name}{String. Name of description table} + +\item{field_descriptions}{Character vector. Descriptions of metadata table fields. If NA, DCMI terms will be used where possible.} + +\item{type}{Character vector. Column types for metadata table fields. see \url{https://airtable.com/developers/web/api/field-model}} + +\item{options}{Data frame. Options for fields in metadata table.} +} +\value{ +List. Outputs from creating the table and inserting the records +} +\description{ +Create the descriptive metadata table for the base +} +\details{ +DCMI terms can be found here \url{https://www.dublincore.org/specifications/dublin-core/dcmi-terms/} +} +\examples{ +\dontrun{ +base = "appVjIfAo8AJlfTkx" +table_name= "description" + +description <- air_generate_base_description(title = "Example Base", + creator = "Collin Schwantes") + +air_create_description_table(base,description,table_name) +} + +} diff --git a/man/air_create_field.Rd b/man/air_create_field.Rd new file mode 100644 index 0000000..9e32dba --- /dev/null +++ b/man/air_create_field.Rd @@ -0,0 +1,34 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_create_field} +\alias{air_create_field} +\title{Create a new field in a table} +\usage{ +air_create_field( + base, + table_id, + name, + description = NA, + type = "singleLineText", + options = NA +) +} +\arguments{ +\item{base}{String. Base id} + +\item{table_id}{String. Table id. Can be found using \code{air_get_schema}} + +\item{name}{String. Name of the field} + +\item{description}{String. Description of the field} + +\item{type}{String. Type of field. See https://airtable.com/developers/web/api/field-model} + +\item{options}{Data frame. See https://airtable.com/developers/web/api/field-model} +} +\value{ +description of newly created field as a list +} +\description{ +See https://airtable.com/developers/web/api/create-field +} diff --git a/man/air_create_metadata_table.Rd b/man/air_create_metadata_table.Rd new file mode 100644 index 0000000..dfaa5ea --- /dev/null +++ b/man/air_create_metadata_table.Rd @@ -0,0 +1,34 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_create_metadata_table} +\alias{air_create_metadata_table} +\title{Create a new metadata table in the base} +\usage{ +air_create_metadata_table( + base, + meta_data, + table_name = "Meta Data", + field_descriptions = NA, + type = "singleLineText", + options = NA +) +} +\arguments{ +\item{base}{String. Base id} + +\item{meta_data}{Data frame. Contains metadata records. From air_generate_metadata*} + +\item{table_name}{String. name of the metadata table. default is "Meta Data"} + +\item{field_descriptions}{Character vector. Descriptions of metadata table fields} + +\item{type}{Character vector. Column types for metadata table fields. see https://airtable.com/developers/web/api/field-model} + +\item{options}{Data frame. Options for fields in metadata table.} +} +\value{ +List with outcome from creating the table and inserting the records +} +\description{ +Create a new metadata table in the base +} diff --git a/man/air_create_table.Rd b/man/air_create_table.Rd new file mode 100644 index 0000000..7f37a87 --- /dev/null +++ b/man/air_create_table.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_create_table} +\alias{air_create_table} +\title{A function to create new tables in a base} +\usage{ +air_create_table(base, table_list) +} +\arguments{ +\item{base}{String. ID for the base} + +\item{table_list}{List. see \code{air_table_template}} +} +\value{ +Data frame of table schema +} +\description{ +Takes a list object with appropriate arguments (see \code{air_table_template}) +converts it to JSON then adds it to the specified base. +} +\note{ +See https://airtable.com/developers/web/api/create-table +} diff --git a/man/air_dump_to_csv.Rd b/man/air_dump_to_csv.Rd index 1d59dc6..37cf2e1 100644 --- a/man/air_dump_to_csv.Rd +++ b/man/air_dump_to_csv.Rd @@ -14,7 +14,7 @@ air_dump_to_csv(table_list, output_dir = "outputs", overwrite = FALSE) \item{overwrite}{Logical. Should outputs be overwritten if they already exist?} } \value{ -list. Returns the table_list object +Vector of file paths } \description{ Saves data.frames from air_dump to csv files. File names are determined by diff --git a/man/air_fields_df_template.Rd b/man/air_fields_df_template.Rd new file mode 100644 index 0000000..4a9a483 --- /dev/null +++ b/man/air_fields_df_template.Rd @@ -0,0 +1,14 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_fields_df_template} +\alias{air_fields_df_template} +\title{Template for for creating a table from a dataframe} +\usage{ +air_fields_df_template(name, description, type, options = NA) +} +\arguments{ +\item{options}{} +} +\description{ +Template for for creating a table from a dataframe +} diff --git a/man/air_generate_base_description.Rd b/man/air_generate_base_description.Rd index 3291517..703c48c 100644 --- a/man/air_generate_base_description.Rd +++ b/man/air_generate_base_description.Rd @@ -6,9 +6,14 @@ \usage{ air_generate_base_description( title = NA, + creator = NA, + created = NA, primary_contact = NA, email = NA, - base_description = NA, + description = NA, + contributor = NA, + identifier = NA, + license = NA, ... ) } @@ -16,19 +21,27 @@ air_generate_base_description( \item{title}{String. Title is a property that refers to the name or names by which a resource is formally known.} +\item{creator}{String. Person or people who created the base} + \item{primary_contact}{String. Person or entity primarily responsible for making the content of a resource} \item{email}{String. Email of primary_contact} -\item{base_description}{String. This property refers to the description of +\item{description}{String. This property refers to the description of the content of a resource. The description is a potentially rich source of indexable terms and assist the users in their selection of an appropriate resource.} +\item{contributor}{String. An entity responsible for making contributions to the resource.} + +\item{identifier}{String. An unambiguous reference to the resource within a given context.} + +\item{license}{String. A legal document giving official permission to do something with the resource. "CC BY 4.0"} + \item{...}{String. Additional descriptive metadata elements. See details. Additional elements can be added as name pair values e.g. -\code{license = "CC BY 4.0", is_part_of = "https://doi.org/10.48321/MyDMP01"}} +\code{ isPartOf = "https://doi.org/00.00000/MyPaper01", isReferencedBy = "https://doi.org/10.48321/MyDMP01"}} } \value{ data.frame with descriptive metadata diff --git a/man/air_get.Rd b/man/air_get.Rd index c5514f2..37a501a 100644 --- a/man/air_get.Rd +++ b/man/air_get.Rd @@ -24,7 +24,7 @@ air_get( \item{table_name}{Table name} \item{record_id}{(optional) Use record ID argument to retrieve an existing -record details} +record details. See \url{https://airtable.com/developers/web/api/get-record}} \item{limit}{(optional) A limit on the number of records to be returned. Limit can range between 1 and 100.} @@ -53,6 +53,10 @@ A data frame with records or a list with record details if \code{record_id} is specified. } \description{ +Retrieve records or a single record from a table. If you provide a record_id, +you cannot specify fields, views, or filterFormulas. +} +\details{ You can retrieve records in an order of a view by providing the name or ID of the view in the view query parameter. The results will include only records visible in the order they are displayed. diff --git a/man/air_get_base_description_from_table.Rd b/man/air_get_base_description_from_table.Rd index a140403..308f602 100644 --- a/man/air_get_base_description_from_table.Rd +++ b/man/air_get_base_description_from_table.Rd @@ -4,7 +4,11 @@ \alias{air_get_base_description_from_table} \title{Get base description from table} \usage{ -air_get_base_description_from_table(base, table_name) +air_get_base_description_from_table( + base, + table_name, + field_names_to_snakecase = TRUE +) } \arguments{ \item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} diff --git a/man/air_get_enterprise.Rd b/man/air_get_enterprise.Rd new file mode 100644 index 0000000..3239fd0 --- /dev/null +++ b/man/air_get_enterprise.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_get_enterprise.R +\name{air_get_enterprise} +\alias{air_get_enterprise} +\title{Get data from the enterprise endpoint} +\usage{ +air_get_enterprise(path, id, params = NULL) +} +\arguments{ +\item{path}{String. What part of the metadata endpoint are you querying? +See \url{https://airtable.com/developers/web/api/introduction}} + +\item{id}{String. ID of item to be queried. e.g. workspace id or enterprise id. +If id = NULL then the id will be omitted from the request url.} + +\item{params}{Optional. List. List of parameters} +} +\value{ +list from jsonlite +} +\description{ +Get data from the enterprise endpoint +} diff --git a/man/air_get_schema.Rd b/man/air_get_schema.Rd index ab7a0e3..0abb1b4 100644 --- a/man/air_get_schema.Rd +++ b/man/air_get_schema.Rd @@ -1,9 +1,11 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/air_get_schema.R +% Please edit documentation in R/air_get_schema.R, R/air_metadata_api.R \name{air_get_schema} \alias{air_get_schema} \title{Get base schema} \usage{ +air_get_schema(base, ...) + air_get_schema(base, ...) } \arguments{ @@ -12,14 +14,22 @@ air_get_schema(base, ...) \item{...}{additional paramters} } \value{ +list of schema + list of schema } \description{ +Get the schema for the tables in a base. This is a wrapper for the api call +Get base schema. + Get the schema for the tables in a base. This is a wrapper for the api call Get base schema. } \section{Using Metadata API}{ +Metadata api is currently available to all users. + + Metadata api is currently available to all users. } diff --git a/man/air_list_bases.Rd b/man/air_list_bases.Rd new file mode 100644 index 0000000..ee1dbf8 --- /dev/null +++ b/man/air_list_bases.Rd @@ -0,0 +1,14 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_list_bases} +\alias{air_list_bases} +\title{Get list of bases for an Token} +\usage{ +air_list_bases(request_url = "https://api.airtable.com/v0/meta/bases") +} +\value{ +list +} +\description{ +Get list of bases for an Token +} diff --git a/man/air_table_template.Rd b/man/air_table_template.Rd new file mode 100644 index 0000000..3e69092 --- /dev/null +++ b/man/air_table_template.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_table_template} +\alias{air_table_template} +\title{Template for lists that describe tables in Airtable} +\usage{ +air_table_template(table_name, description, fields_df) +} +\arguments{ +\item{table_name}{String. Name of table} + +\item{description}{String. Description of the table} + +\item{fields_df}{Data frame. Data frame describing the field in a table. +Should contain a name, description,type, and options field. +if} +} +\value{ +List with table name, description, and fields +} +\description{ +Template for lists that describe tables in Airtable +} diff --git a/man/air_update_description_table.Rd b/man/air_update_description_table.Rd new file mode 100644 index 0000000..811da02 --- /dev/null +++ b/man/air_update_description_table.Rd @@ -0,0 +1,31 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_update_description_table} +\alias{air_update_description_table} +\title{Update the description table} +\usage{ +air_update_description_table( + base, + description, + table_name = "Description", + join_field = "title", + record_id_field = "id" +) +} +\arguments{ +\item{base}{String. Base id} + +\item{description}{Data frame. Contains updated description} + +\item{table_name}{String. Name of description table} + +\item{join_field}{String. Field to perform join on} + +\item{record_id_field}{String. Name of the record id field} +} +\value{ +list that logs updates +} +\description{ +Update the descriptive metadata table in airtable +} diff --git a/man/air_update_metadata_table.Rd b/man/air_update_metadata_table.Rd new file mode 100644 index 0000000..a145200 --- /dev/null +++ b/man/air_update_metadata_table.Rd @@ -0,0 +1,36 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_update_metadata_table} +\alias{air_update_metadata_table} +\title{Update the structural metadata table} +\usage{ +air_update_metadata_table( + base, + meta_data, + table_name = "Meta Data", + join_field = "field_id", + record_id_field = "id" +) +} +\arguments{ +\item{base}{String. Base id} + +\item{meta_data}{Data frame. Contains metadata records. From air_generate_metadata*} + +\item{table_name}{String. Name of metadata table} + +\item{join_field}{String. Name of field to join new and current metadata. Likely \code{field_id}} + +\item{record_id_field}{String. Name of record id field. Like \code{id}} +} +\description{ +Update the structural metadata table +} +\examples{ +\dontrun{ +base = "appVjIfAo8AJlfTkx" +metadata <- air_generate_metadata_from_api(base = base) +air_update_metadata_table(base,metadata) +} + +} From bd8875910198798366f1d187522adb260027100d Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 6 Apr 2023 08:59:43 -0600 Subject: [PATCH 079/126] added example --- R/air_download_attachments.R | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/R/air_download_attachments.R b/R/air_download_attachments.R index 8082144..cadfbbe 100644 --- a/R/air_download_attachments.R +++ b/R/air_download_attachments.R @@ -2,7 +2,8 @@ #' #' Download an attachment stored in air tables. Returns original dataframe #' with an additional field called attachment_file_paths. The attachment_file_paths -#' field is of class list so it can handle multiple attachments per record. +#' field is of class list so it can handle multiple attachments per record. File +#' paths are prepended with record ids so that all file names are unique. #' #' @param x Data frame. Output from air_get or fetch_all. #' @param field String. Name of field with file attachments in base @@ -14,6 +15,21 @@ #' @export air_download_attachments #' #' @examples +#' \dontrun{ +#' +#' base <- "appXXXXXXXXX" +#' table_name <- "Table With Attachments" +#' +#' table_original <- air_get(base,table_name) +#' +#' table_with_file_paths <- air_download_attachments(x = table_with_attachments, +#' field = "attachment_field", +#' dir_name = "downloads") +#' +#' table_with_file_paths$attachment_file_paths +#' +#' } +#' air_download_attachments <- function(x, field, dir_name = "downloads",...){ #browser() From 745069a591a85ca405eed1cc6f89dbc8831a1b9d Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 6 Apr 2023 09:00:15 -0600 Subject: [PATCH 080/126] updated namespace --- NAMESPACE | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/NAMESPACE b/NAMESPACE index 3900371..e6db247 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,27 +1,37 @@ # Generated by roxygen2: do not edit by hand S3method(print,airtable.base) +export(air_create_description_table) +export(air_create_field) +export(air_create_metadata_table) +export(air_create_table) export(air_delete) export(air_download_attachments) export(air_dump) export(air_dump_to_csv) export(air_dump_to_json) export(air_expand_csv_arrays) +export(air_fields_df_template) export(air_generate_base_description) export(air_generate_metadata) export(air_get) export(air_get_attachments) export(air_get_base_description_from_table) +export(air_get_enterprise) export(air_get_json) export(air_get_metadata_from_table) export(air_get_schema) export(air_insert) export(air_insert_data_frame) +export(air_list_bases) export(air_make_json) export(air_make_request) export(air_select) +export(air_table_template) export(air_update) export(air_update_data_frame) +export(air_update_description_table) +export(air_update_metadata_table) export(airtable) export(fetch_all) export(fetch_all_json) From 4f23d7ee8a55d36e6b91fc5fee85c5399f2a9d3d Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 7 Apr 2023 10:29:21 -0600 Subject: [PATCH 081/126] updated documentation --- R/air_dump.R | 212 ++++++++++++------- R/air_get_attachments.R | 13 +- R/air_get_json.R | 8 + R/air_metadata_api.R | 227 ++++++++++++++++++++- R/airtabler.R | 9 +- R/get_unique_field_values.R | 1 - man/air_create_field.Rd | 20 ++ man/air_create_metadata_table.Rd | 18 +- man/air_create_table.Rd | 45 ++++ man/air_download_attachments.Rd | 20 +- man/air_dump_to_json.Rd | 15 +- man/air_fields_df_template.Rd | 61 +++++- man/air_fields_list_from_template.Rd | 60 ++++++ man/air_generate_metadata_from_api.Rd | 34 +++ man/air_get_attachments.Rd | 13 ++ man/air_get_base_description_from_table.Rd | 7 + man/air_get_enterprise.Rd | 23 --- man/air_list_bases.Rd | 9 +- man/air_table_template.Rd | 47 +++++ man/air_update_description_table.Rd | 18 ++ man/air_update_metadata_table.Rd | 3 + man/fetch_all_json.Rd | 11 + 22 files changed, 744 insertions(+), 130 deletions(-) create mode 100644 man/air_fields_list_from_template.Rd create mode 100644 man/air_generate_metadata_from_api.Rd delete mode 100644 man/air_get_enterprise.Rd diff --git a/R/air_dump.R b/R/air_dump.R index 3c4b8a2..9f3c43d 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -36,8 +36,8 @@ set_diff <- function(x,y){ return(diff) } -# add the metadata table to the base -#' Create a new metadata table in the base + +#' Create a new structural metadata table in the base #' #' @param base String. Base id #' @param meta_data Data frame. Contains metadata records. From air_generate_metadata* @@ -50,6 +50,18 @@ set_diff <- function(x,y){ #' @export air_create_metadata_table #' #' @examples +#'\dontrun{ +#' # set base id +#' base <- "appXXXXXXXX" +#' # create metadata from api +#' metadata <- air_generate_metadata_from_api(base) +#' # add Meta Data table to base -- will not work if base already has a metadata +#' # table +#' log <- air_create_metadata_table(base,metadata) +#' +#'} +#' +#' air_create_metadata_table <- function(base,meta_data,table_name = "Meta Data", field_descriptions = NA, type = "singleLineText", options = NA){ @@ -257,8 +269,8 @@ air_create_description_table <- function(base, #' @param join_field String. Name of field to join new and current metadata. Likely \code{field_id} #' @param record_id_field String. Name of record id field. Like \code{id} #' -#' @return -#' @export +#' @return List. Log of results for updating metadata +#' @export air_update_metadata_table #' #' @examples #'\dontrun{ @@ -374,9 +386,25 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j #' @param record_id_field String. Name of the record id field #' #' @return list that logs updates -#' @export +#' @export air_update_description_table #' #' @examples +#' +#' \dontrun{ +#' +#' base <- "appXXXXXXXX" +#' table_name <- "Description" +#' # get description from table +#' description <- air_get_base_description_from_table(base, table_name) +#' # update the identifier field +#' description$identifier <- "fake.doi.xyz/029940" +#' # update the table +#' air_update_description_table(base,description) +#' +#' +#' } +#' +#' air_update_description_table <- function(base,description, table_name = "Description", join_field = "title", record_id_field = "id"){ @@ -544,6 +572,24 @@ air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, f # pull data from api and populate metadata table +#' Generate structural metadata from the api +#' +#' @param base String. Base id +#' @param metadata_table_name String. Name of exisiting structural metadata table if it exists +#' @param include_metadata_table Logical. Should the structural metadata table be included in the metadata? +#' +#' @return A data frame with metadata +#' @export air_generate_metadata_from_api +#' +#' @examples +#' +#' \dontrun{ +#' +#' base <- "appXXXXXXXX" +#' metadata <- air_generate_metadata_from_api(base) +#' +#' } + air_generate_metadata_from_api <- function(base, metadata_table_name = "Meta Data", include_metadata_table = FALSE){ @@ -645,9 +691,8 @@ air_generate_metadata_from_api <- function(base, #' @export air_generate_metadata <- function(base, table_names,limit=1){ - warning('Airtable does not return fields with empty values - "", false, or []. - It is better to create a specific metdata table and - parse that with air_get_metadata_*') + warning('For more complete results, use air_generate_metadata_from_api. + Airtable does not return fields with empty values - "", false, or [].') meta_data_table <- purrr::map_dfr(table_names,function(x){ table_x <- airtabler::air_get(base,x,limit = limit ) fields_x <- names(table_x) @@ -673,8 +718,14 @@ air_generate_metadata <- function(base, table_names,limit=1){ #' describes the base and provides attribution #' #' @return data.frame with descriptive metadata. -#' @export - +#' @export air_get_base_description_from_table +#' +#' @examples +#' \dontrun{ +#' base <- "appXXXXXXXX" +#' table_name <- "Description" +#' air_get_base_description_from_table(base, table_name) +#' } air_get_base_description_from_table<- function(base, table_name,field_names_to_snakecase = TRUE){ #fetch table desc_table <- airtabler::fetch_all(base,table_name) @@ -912,9 +963,9 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR } } - named_description_post <- grepl(pattern = "description",x = names(table_list), ignore.case = TRUE) - - table_list[named_description_post][[1]]$created <- Sys.Date() +# named_description_post <- grepl(pattern = "description",x = names(table_list), ignore.case = TRUE) +# +# table_list[named_description_post][[1]]$created <- Sys.Date() return(table_list) } @@ -1069,78 +1120,85 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) #' Dump all tables from a base into json files #' -#' Essentially air_get without converting to Rs +#' Essentially air_get without converting to Rs. Does not add fields with empty +#' values. #' #' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' #' @param metadata Data.frame.Data frame with structural metadata - describes relationship between tables and fields. #' @param description Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. -#' Can be left as NULL if base already contains a table called description. -#' @param add_missing_fields Logical. If true add in missing fields +#' Can be left as NULL if base already contains a table called description #' #' @return List of data.frames. All tables from metadata plus the #' description and metadata tables. -#' @export +#' @export air_dump_to_json #' -air_dump_to_json <- function(base, metadata, description = NULL, add_missing_fields = TRUE){ - # - # names(metadata) <- snakecase::to_snake_case(names(metadata)) - # - # ## check for required fields - # required_fields <- c("table_name","field_name") - # - # if(!all(required_fields %in% names(metadata))){ - # stop(glue::glue("metadata table must contain the - # following fields: {required_fields}. Note - # that field names are converted to snakecase - # before check.")) - # } - # - # - # base_table_names <- unique(metadata$table_name) - # - # print(base_table_names) - # table_list <- base_table_names |> - # purrr::set_names() |> - # purrr::map(function(x){ - # ## get fields from str_metadata - # - # fields_exp <- metadata[metadata$table_name == x,"field_name"] - # - # ## pull table - add check for blank tables - # x_table <- airtabler::fetch_all(base,x) - # - # if(!is.data.frame(x_table)){ - # x_table <- data.frame(id = character()) - # } - # - # ## add in missing columns if any - # fields_obs <- names(x_table) - # - # # check if any discrepancy between metadata and table - # fields_diff <- set_diff(fields_exp,fields_obs) - # #browser() - # if(!is.null(fields_diff)){ - # # check for fields in obs not in exp - error - # obs_exp <- setdiff(fields_obs,fields_exp) - # ignore_fields <- c("id","createdTime") - # ignore_fields_pattern <- paste(ignore_fields,collapse = "|") - # if(length(obs_exp) != 0 & !all(obs_exp %in% ignore_fields)){ - # missing_fields <- obs_exp[!grepl(ignore_fields_pattern,obs_exp,ignore.case = FALSE)] - # missing_fields_glue <- paste(missing_fields, collapse = ", ") - # stop(glue::glue('The metadata table is missing the following fields from table {x}: - # {missing_fields_glue} - # Please update the metadata table.https://airtable.com/{base}')) - # } - # # check for fields in exp and not in obs - append unless frictionless - # if(add_missing_fields){ - # exp_obs <- setdiff(fields_exp,fields_obs) - # x_table[exp_obs] <- list(character(0)) - # } - # } - # - # return(x_table) - # - # }) +air_dump_to_json <- function(base, metadata, description = NULL, output_dir= "outputs", overwrite = FALSE){ + + names(metadata) <- snakecase::to_snake_case(names(metadata)) + + ## check for required fields + required_fields <- c("table_name") + + if(!all(required_fields %in% names(metadata))){ + stop(glue::glue("metadata table must contain the + following field: {required_fields}. Note + that field names are converted to snakecase + before check.")) + } + + + base_table_names <- unique(metadata$table_name) + + print(base_table_names) + + json_list <- base_table_names |> + purrr::set_names() |> + purrr::map(function(x){ + + ### no expected fields, just json + ### see air_make_json for refactor + + x_json <- fetch_all_json(base,x) + + return(x_json) + + }) + + json_list$metadata <- jsonlite::toJSON(metadata) + + # check for description table + named_description <- grepl(pattern = "description",x = names(json_list), ignore.case = TRUE) + + if(!is.null(description)){ + if(any(named_description)){ + warning("Base has a description table and a description data.frame was supplied to + this function. Inserting description data.frame at $description. Table + extract may be overwritten.") + } + json_list$description <- jsonlite::toJSON(description) + } + + ## does not add a description table if not present + + + ## write to files + output_id <- rlang::hash(json_list) + + output_dir_path <- sprintf("%s/%s",output_dir,output_id) + + dir.create(output_dir_path,recursive = TRUE) + + purrr::walk2(json_list, names(json_list), function(x_table,y_table_name){ + + output_file_path <- sprintf("%s/%s.json",output_dir_path,y_table_name) + + jsonlite::write_json(x_table,path = output_file_path) + }) + + return(list.files(output_dir_path,full.names = TRUE)) + + + } diff --git a/R/air_get_attachments.R b/R/air_get_attachments.R index 1003ef4..eb5b48c 100644 --- a/R/air_get_attachments.R +++ b/R/air_get_attachments.R @@ -19,8 +19,19 @@ #' @export air_get_attachments #' #' @examples +#' +#' \dontrun{ +#' +#' base <- "appXXXXXXXXX" +#' table_name <- "table with excel attachments" +#' +#' table_with_attachments <- air_get_attachments(base,table_name, field = "attachment_field" ) +#' +#' } +#' +#' air_get_attachments <- function(base, table_name, field, download_file = FALSE, dir_name = "downloads", extract_type ="excel", extract_field ="excel_extract", skip = 0, parse_all_sheets = FALSE, ...){ - #browser() + # get data x <- fetch_all(base,table_name,...) diff --git a/R/air_get_json.R b/R/air_get_json.R index 85d9762..2dc339e 100644 --- a/R/air_get_json.R +++ b/R/air_get_json.R @@ -80,6 +80,14 @@ air_get_json <- function(base, table_name, #' #' @examples #' +#' \dontrun{ +#' base <- "appXXXXXXX" +#' table_name <- "My Table" +#' +#' fetch_all_json(base, table_name) +#' +#' } +#' fetch_all_json <- function(base, table_name, ...) { out <- list() out[[1]] <- air_get_json(base, table_name, combined_result = FALSE,...) diff --git a/R/air_metadata_api.R b/R/air_metadata_api.R index 079cc9b..2d5f44f 100644 --- a/R/air_metadata_api.R +++ b/R/air_metadata_api.R @@ -44,18 +44,63 @@ type_option_map <-function(){ } -#' Template for for creating a table from a dataframe +#' Template for for creating a table from a tibble #' -#' @param name -#' @param description -#' @param type -#' @param options +#' Convenience function for creating the content of tables that will created or +#' updated viaAPI. #' -#' @return -#' @export +#' @param name String. Names of fields in the table +#' @param description String. Descriptions of fields +#' @param type String. Type of columns. For values see \url{https://airtable.com/developers/web/api/model/field-type} +#' @param options List. Options will be converted from lists to JSON. For field options see \url{https://airtable.com/developers/web/api/field-model} +#' +#' @return Tibble with attributes required for fields in a table +#' @export air_fields_df_template #' #' @examples #' +#'\dontrun{ +#' base <- "appQ94sELAtFnXPxx" +#' +#' base_schema <- air_get_schema(base) +#' +#' tables<- base_schema$tables +#' +#' field_names <- c("Planet","Chapter","Book", "Known Inhabitants") +#' +#' field_desc <- c("Name of planet in Foundation Series", +#' "Chapters where planet is referenced", +#' "Books where planet is referenced", +#' "Characters mentioned as living on or being from that planet") +#' +#' field_types <- c("singleLineText",rep("multipleRecordLinks",3)) +#' +#' field_options <- c(NA,list( +#' list( +#' linkedTableId = tables[tables$name == "Chapter","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Book","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Character","id"] +#' ) +#' ) +#' ) +#' +#' field_df<- air_fields_df_template(name = field_names, +#' description = field_desc, +#' type = field_types, +#' options = field_options) +#' +#' table_list <- air_table_template(table_name = "Planet",description = "Planets of Foundation",fields_df = field_tables) +#' +#' air_create_table(base, table_list) +#'} air_fields_df_template <- function(name,description, type, options = NA){ df <- tibble::tibble(name = name, description = description, @@ -65,6 +110,57 @@ air_fields_df_template <- function(name,description, type, options = NA){ return(df) } +#' Convert field data frame to list +#' +#' Converts the field data frame to a list of easier translation to JSON +#' +#' @param df Data frame. From air_fields_df_template +#' +#' @return List. Structured for easy parsing into JSON +#' @export air_fields_list_from_template +#' +#' @examples +#'\dontrun{ +#' base <- "appQ94sELAtFnXPxx" +#' +#' base_schema <- air_get_schema(base) +#' +#' tables<- base_schema$tables +#' +#' field_names <- c("Planet","Chapter","Book", "Known Inhabitants") +#' +#' field_desc <- c("Name of planet in Foundation Series", +#' "Chapters where planet is referenced", +#' "Books where planet is referenced", +#' "Characters mentioned as living on or being from that planet") +#' +#' field_types <- c("singleLineText",rep("multipleRecordLinks",3)) +#' +#' field_options <- c(NA,list( +#' list( +#' linkedTableId = tables[tables$name == "Chapter","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Book","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Character","id"] +#' ) +#' ) +#' ) +#' +#' field_df <- air_fields_df_template(name = field_names, +#' description = field_desc, +#' type = field_types, +#' options = field_options) +#' +#' fields_list <- air_fields_list_from_template(df = fields_df) +#' +#'} air_fields_list_from_template <- function(df){ ## create a list of field objects purrr::pmap(df, function(name, @@ -99,6 +195,51 @@ air_fields_list_from_template <- function(df){ #' @export air_table_template #' #' @examples +#' +#' #'\dontrun{ +#' base <- "appQ94sELAtFnXPxx" +#' +#' base_schema <- air_get_schema(base) +#' +#' tables<- base_schema$tables +#' +#' field_names <- c("Planet","Chapter","Book", "Known Inhabitants") +#' +#' field_desc <- c("Name of planet in Foundation Series", +#' "Chapters where planet is referenced", +#' "Books where planet is referenced", +#' "Characters mentioned as living on or being from that planet") +#' +#' field_types <- c("singleLineText",rep("multipleRecordLinks",3)) +#' +#' field_options <- c(NA,list( +#' list( +#' linkedTableId = tables[tables$name == "Chapter","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Book","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Character","id"] +#' ) +#' ) +#' ) +#' +#' field_df<- air_fields_df_template(name = field_names, +#' description = field_desc, +#' type = field_types, +#' options = field_options) +#' +#' table_list <- air_table_template(table_name = "Planet",description = "Planets of Foundation",fields_df = field_tables) +#' +#' air_create_table(base, table_list) +#'} +#' +#' air_table_template <- function(table_name, description, fields_df ){ valid_cols <- c("description","name","type","options") @@ -155,6 +296,49 @@ air_table_template <- function(table_name, description, fields_df ){ #' @export air_create_table #' #' @examples +#' +#' #'\dontrun{ +#' base <- "appQ94sELAtFnXPxx" +#' +#' base_schema <- air_get_schema(base) +#' +#' tables<- base_schema$tables +#' +#' field_names <- c("Planet","Chapter","Book", "Known Inhabitants") +#' +#' field_desc <- c("Name of planet in Foundation Series", +#' "Chapters where planet is referenced", +#' "Books where planet is referenced", +#' "Characters mentioned as living on or being from that planet") +#' +#' field_types <- c("singleLineText",rep("multipleRecordLinks",3)) +#' +#' field_options <- c(NA,list( +#' list( +#' linkedTableId = tables[tables$name == "Chapter","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Book","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Character","id"] +#' ) +#' ) +#' ) +#' +#' field_df<- air_fields_df_template(name = field_names, +#' description = field_desc, +#' type = field_types, +#' options = field_options) +#' +#' table_list <- air_table_template(table_name = "Planet",description = "Planets of Foundation",fields_df = field_tables) +#' +#' air_create_table(base, table_list) +#'} air_create_table <- function(base, table_list){ request_url <- sprintf("%s/%s/tables", air_meta_url, base) request_url <- utils::URLencode(request_url) @@ -196,6 +380,24 @@ air_create_table <- function(base, table_list){ #' @export air_create_field #' #' @examples +#'\donotrun{ +#' base_schema <- air_get_schema(base) +#' +#' base_schema$tables +#' +#' air_create_field(base,table_id = base_schema$tables$id[[4]], +#' name = "Has Nucleics", +#' description = "Logical. Does this planet have nucleics?", +#' type = "checkbox", +#' options = list( +#' list( +#' "color"= "greenBright", +#' "icon"= "check" +#' ) +#' ) +#' ) +#' } +#' air_create_field <- function(base, table_id, name, @@ -243,15 +445,20 @@ air_create_field <- function(base, #' Get list of bases for an Token #' -#' @return list -#' @export +#' @return list. List of bases a token can access. +#' @export air_list_bases +#' #' #' @examples +#' +#' \dontrun{ +#' air_list_bases() +#' } +#' air_list_bases <- function(request_url = "https://api.airtable.com/v0/meta/bases"){ request_url <- utils::URLencode(request_url) - # call service: res <- httr::GET( request_url, diff --git a/R/airtabler.R b/R/airtabler.R index 0ef3b66..7fcc667 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -344,9 +344,8 @@ air_insert <- function(base, table_name, record_data) { #' allows you to add new options to select type fields. #' #' @return JSON with record data -#' @export +#' @export air_make_json #' -#' @examples air_make_json <- function (base, table_name, record_data, record_id = NULL, method = "POST",typecast = TRUE){ if (inherits(record_data, "data.frame")) { return(air_insert_data_frame(base, table_name, record_data, typecast)) @@ -401,9 +400,8 @@ air_make_json <- function (base, table_name, record_data, record_id = NULL, meth #' @param method String. One of "POST", "PATCH", or "DELETE" #' #' @return Status of HTTP request -#' @export +#' @export air_make_request #' -#' @examples air_make_request <- function(base, table_name, json_record_data, record_id = NULL, method = c("POST","PATCH","DELETE")){ if(method == "POST"){ @@ -484,9 +482,8 @@ air_insert_data_frame <- function(base, table_name, records,typecast) { #' @param records Dataframe. Values to update #' #' @return Status of HTTP request -#' @export +#' @export air_update_data_frame #' -#' @examples air_update_data_frame <- function(base, table_name, record_ids, records) { lapply(seq_len(nrow(records)), function(i) { record_data <- as.list(records[i,]) diff --git a/R/get_unique_field_values.R b/R/get_unique_field_values.R index ea3b990..8c82429 100644 --- a/R/get_unique_field_values.R +++ b/R/get_unique_field_values.R @@ -15,7 +15,6 @@ #' @return vector of unique values #' @export #' -#' @examples get_unique_field_values <- function(base, table_name, fields){ diff --git a/man/air_create_field.Rd b/man/air_create_field.Rd index 9e32dba..3d55165 100644 --- a/man/air_create_field.Rd +++ b/man/air_create_field.Rd @@ -32,3 +32,23 @@ description of newly created field as a list \description{ See https://airtable.com/developers/web/api/create-field } +\examples{ +\donotrun{ +base_schema <- air_get_schema(base) + +base_schema$tables + +air_create_field(base,table_id = base_schema$tables$id[[4]], + name = "Has Nucleics", + description = "Logical. Does this planet have nucleics?", + type = "checkbox", + options = list( + list( + "color"= "greenBright", + "icon"= "check" + ) + ) +) +} + +} diff --git a/man/air_create_metadata_table.Rd b/man/air_create_metadata_table.Rd index dfaa5ea..2829742 100644 --- a/man/air_create_metadata_table.Rd +++ b/man/air_create_metadata_table.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/air_dump.R \name{air_create_metadata_table} \alias{air_create_metadata_table} -\title{Create a new metadata table in the base} +\title{Create a new structural metadata table in the base} \usage{ air_create_metadata_table( base, @@ -30,5 +30,19 @@ air_create_metadata_table( List with outcome from creating the table and inserting the records } \description{ -Create a new metadata table in the base +Create a new structural metadata table in the base +} +\examples{ +\dontrun{ +# set base id +base <- "appXXXXXXXX" +# create metadata from api +metadata <- air_generate_metadata_from_api(base) +# add Meta Data table to base -- will not work if base already has a metadata +# table +log <- air_create_metadata_table(base,metadata) + +} + + } diff --git a/man/air_create_table.Rd b/man/air_create_table.Rd index 7f37a87..608e4ed 100644 --- a/man/air_create_table.Rd +++ b/man/air_create_table.Rd @@ -21,3 +21,48 @@ converts it to JSON then adds it to the specified base. \note{ See https://airtable.com/developers/web/api/create-table } +\examples{ + +#'\dontrun{ +base <- "appQ94sELAtFnXPxx" + +base_schema <- air_get_schema(base) + +tables<- base_schema$tables + +field_names <- c("Planet","Chapter","Book", "Known Inhabitants") + +field_desc <- c("Name of planet in Foundation Series", + "Chapters where planet is referenced", + "Books where planet is referenced", + "Characters mentioned as living on or being from that planet") + +field_types <- c("singleLineText",rep("multipleRecordLinks",3)) + +field_options <- c(NA,list( + list( + linkedTableId = tables[tables$name == "Chapter","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Book","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Character","id"] + ) +) +) + +field_df<- air_fields_df_template(name = field_names, + description = field_desc, + type = field_types, + options = field_options) + +table_list <- air_table_template(table_name = "Planet",description = "Planets of Foundation",fields_df = field_tables) + +air_create_table(base, table_list) +} +} diff --git a/man/air_download_attachments.Rd b/man/air_download_attachments.Rd index b4aad82..7b467cc 100644 --- a/man/air_download_attachments.Rd +++ b/man/air_download_attachments.Rd @@ -22,5 +22,23 @@ Returns x with an additional field called attachment_file_paths \description{ Download an attachment stored in air tables. Returns original dataframe with an additional field called attachment_file_paths. The attachment_file_paths -field is of class list so it can handle multiple attachments per record. +field is of class list so it can handle multiple attachments per record. File +paths are prepended with record ids so that all file names are unique. +} +\examples{ +\dontrun{ + +base <- "appXXXXXXXXX" +table_name <- "Table With Attachments" + +table_original <- air_get(base,table_name) + +table_with_file_paths <- air_download_attachments(x = table_with_attachments, + field = "attachment_field", + dir_name = "downloads") + +table_with_file_paths$attachment_file_paths + +} + } diff --git a/man/air_dump_to_json.Rd b/man/air_dump_to_json.Rd index 90467d4..99814ad 100644 --- a/man/air_dump_to_json.Rd +++ b/man/air_dump_to_json.Rd @@ -4,7 +4,13 @@ \alias{air_dump_to_json} \title{Dump all tables from a base into json files} \usage{ -air_dump_to_json(base, metadata, description = NULL, add_missing_fields = TRUE) +air_dump_to_json( + base, + metadata, + description = NULL, + output_dir = "outputs", + overwrite = FALSE +) } \arguments{ \item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} @@ -12,14 +18,13 @@ air_dump_to_json(base, metadata, description = NULL, add_missing_fields = TRUE) \item{metadata}{Data.frame.Data frame with structural metadata - describes relationship between tables and fields.} \item{description}{Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. -Can be left as NULL if base already contains a table called description.} - -\item{add_missing_fields}{Logical. If true add in missing fields} +Can be left as NULL if base already contains a table called description} } \value{ List of data.frames. All tables from metadata plus the description and metadata tables. } \description{ -Essentially air_get without converting to Rs +Essentially air_get without converting to Rs. Does not add fields with empty +values. } diff --git a/man/air_fields_df_template.Rd b/man/air_fields_df_template.Rd index 4a9a483..6b522f3 100644 --- a/man/air_fields_df_template.Rd +++ b/man/air_fields_df_template.Rd @@ -2,13 +2,68 @@ % Please edit documentation in R/air_metadata_api.R \name{air_fields_df_template} \alias{air_fields_df_template} -\title{Template for for creating a table from a dataframe} +\title{Template for for creating a table from a tibble} \usage{ air_fields_df_template(name, description, type, options = NA) } \arguments{ -\item{options}{} +\item{name}{String. Names of fields in the table} + +\item{description}{String. Descriptions of fields} + +\item{type}{String. Type of columns. For values see \url{https://airtable.com/developers/web/api/model/field-type}} + +\item{options}{List. Options will be converted from lists to JSON. For field options see \url{https://airtable.com/developers/web/api/field-model}} +} +\value{ +Tibble with attributes required for fields in a table } \description{ -Template for for creating a table from a dataframe +Convenience function for creating the content of tables that will created or +updated viaAPI. +} +\examples{ + +\dontrun{ +base <- "appQ94sELAtFnXPxx" + +base_schema <- air_get_schema(base) + +tables<- base_schema$tables + +field_names <- c("Planet","Chapter","Book", "Known Inhabitants") + +field_desc <- c("Name of planet in Foundation Series", + "Chapters where planet is referenced", + "Books where planet is referenced", + "Characters mentioned as living on or being from that planet") + +field_types <- c("singleLineText",rep("multipleRecordLinks",3)) + +field_options <- c(NA,list( + list( + linkedTableId = tables[tables$name == "Chapter","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Book","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Character","id"] + ) +) +) + +field_df<- air_fields_df_template(name = field_names, + description = field_desc, + type = field_types, + options = field_options) + +table_list <- air_table_template(table_name = "Planet",description = "Planets of Foundation",fields_df = field_tables) + +air_create_table(base, table_list) +} } diff --git a/man/air_fields_list_from_template.Rd b/man/air_fields_list_from_template.Rd new file mode 100644 index 0000000..d7fac4d --- /dev/null +++ b/man/air_fields_list_from_template.Rd @@ -0,0 +1,60 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_fields_list_from_template} +\alias{air_fields_list_from_template} +\title{Convert field data frame to list} +\usage{ +air_fields_list_from_template(df) +} +\arguments{ +\item{df}{Data frame. From air_fields_df_template} +} +\value{ +List. Structured for easy parsing into JSON +} +\description{ +Converts the field data frame to a list of easier translation to JSON +} +\examples{ +\dontrun{ +base <- "appQ94sELAtFnXPxx" + +base_schema <- air_get_schema(base) + +tables<- base_schema$tables + +field_names <- c("Planet","Chapter","Book", "Known Inhabitants") + +field_desc <- c("Name of planet in Foundation Series", + "Chapters where planet is referenced", + "Books where planet is referenced", + "Characters mentioned as living on or being from that planet") + +field_types <- c("singleLineText",rep("multipleRecordLinks",3)) + +field_options <- c(NA,list( + list( + linkedTableId = tables[tables$name == "Chapter","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Book","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Character","id"] + ) +) +) + +field_df <- air_fields_df_template(name = field_names, + description = field_desc, + type = field_types, + options = field_options) + +fields_list <- air_fields_list_from_template(df = fields_df) + +} +} diff --git a/man/air_generate_metadata_from_api.Rd b/man/air_generate_metadata_from_api.Rd new file mode 100644 index 0000000..5692d4f --- /dev/null +++ b/man/air_generate_metadata_from_api.Rd @@ -0,0 +1,34 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_generate_metadata_from_api} +\alias{air_generate_metadata_from_api} +\title{Generate structural metadata from the api} +\usage{ +air_generate_metadata_from_api( + base, + metadata_table_name = "Meta Data", + include_metadata_table = FALSE +) +} +\arguments{ +\item{base}{String. Base id} + +\item{metadata_table_name}{String. Name of exisiting structural metadata table if it exists} + +\item{include_metadata_table}{Logical. Should the structural metadata table be included in the metadata?} +} +\value{ +A data frame with metadata +} +\description{ +Generate structural metadata from the api +} +\examples{ + +\dontrun{ + +base <- "appXXXXXXXX" +metadata <- air_generate_metadata_from_api(base) + +} +} diff --git a/man/air_get_attachments.Rd b/man/air_get_attachments.Rd index 3e6db31..6157f62 100644 --- a/man/air_get_attachments.Rd +++ b/man/air_get_attachments.Rd @@ -46,3 +46,16 @@ named list of data frames \description{ Get an attachment stored in air tables. For excel files, returns a named list. } +\examples{ + +\dontrun{ + +base <- "appXXXXXXXXX" +table_name <- "table with excel attachments" + + table_with_attachments <- air_get_attachments(base,table_name, field = "attachment_field" ) + +} + + +} diff --git a/man/air_get_base_description_from_table.Rd b/man/air_get_base_description_from_table.Rd index 308f602..513cf7d 100644 --- a/man/air_get_base_description_from_table.Rd +++ b/man/air_get_base_description_from_table.Rd @@ -24,3 +24,10 @@ Pull a table that has descriptive metadata. Requires the following fields: "title","primary_contact","email","base_description" } +\examples{ +\dontrun{ +base <- "appXXXXXXXX" +table_name <- "Description" +air_get_base_description_from_table(base, table_name) +} +} diff --git a/man/air_get_enterprise.Rd b/man/air_get_enterprise.Rd deleted file mode 100644 index 3239fd0..0000000 --- a/man/air_get_enterprise.Rd +++ /dev/null @@ -1,23 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/air_get_enterprise.R -\name{air_get_enterprise} -\alias{air_get_enterprise} -\title{Get data from the enterprise endpoint} -\usage{ -air_get_enterprise(path, id, params = NULL) -} -\arguments{ -\item{path}{String. What part of the metadata endpoint are you querying? -See \url{https://airtable.com/developers/web/api/introduction}} - -\item{id}{String. ID of item to be queried. e.g. workspace id or enterprise id. -If id = NULL then the id will be omitted from the request url.} - -\item{params}{Optional. List. List of parameters} -} -\value{ -list from jsonlite -} -\description{ -Get data from the enterprise endpoint -} diff --git a/man/air_list_bases.Rd b/man/air_list_bases.Rd index ee1dbf8..d907262 100644 --- a/man/air_list_bases.Rd +++ b/man/air_list_bases.Rd @@ -7,8 +7,15 @@ air_list_bases(request_url = "https://api.airtable.com/v0/meta/bases") } \value{ -list +list. List of bases a token can access. } \description{ Get list of bases for an Token } +\examples{ + +\dontrun{ +air_list_bases() +} + +} diff --git a/man/air_table_template.Rd b/man/air_table_template.Rd index 3e69092..7bcc057 100644 --- a/man/air_table_template.Rd +++ b/man/air_table_template.Rd @@ -21,3 +21,50 @@ List with table name, description, and fields \description{ Template for lists that describe tables in Airtable } +\examples{ + +#'\dontrun{ +base <- "appQ94sELAtFnXPxx" + +base_schema <- air_get_schema(base) + +tables<- base_schema$tables + +field_names <- c("Planet","Chapter","Book", "Known Inhabitants") + +field_desc <- c("Name of planet in Foundation Series", + "Chapters where planet is referenced", + "Books where planet is referenced", + "Characters mentioned as living on or being from that planet") + +field_types <- c("singleLineText",rep("multipleRecordLinks",3)) + +field_options <- c(NA,list( + list( + linkedTableId = tables[tables$name == "Chapter","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Book","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Character","id"] + ) +) +) + +field_df<- air_fields_df_template(name = field_names, + description = field_desc, + type = field_types, + options = field_options) + +table_list <- air_table_template(table_name = "Planet",description = "Planets of Foundation",fields_df = field_tables) + +air_create_table(base, table_list) +} + + +} diff --git a/man/air_update_description_table.Rd b/man/air_update_description_table.Rd index 811da02..6ea947d 100644 --- a/man/air_update_description_table.Rd +++ b/man/air_update_description_table.Rd @@ -29,3 +29,21 @@ list that logs updates \description{ Update the descriptive metadata table in airtable } +\examples{ + +\dontrun{ + +base <- "appXXXXXXXX" +table_name <- "Description" +# get description from table +description <- air_get_base_description_from_table(base, table_name) +# update the identifier field +description$identifier <- "fake.doi.xyz/029940" +# update the table +air_update_description_table(base,description) + + +} + + +} diff --git a/man/air_update_metadata_table.Rd b/man/air_update_metadata_table.Rd index a145200..0015337 100644 --- a/man/air_update_metadata_table.Rd +++ b/man/air_update_metadata_table.Rd @@ -23,6 +23,9 @@ air_update_metadata_table( \item{record_id_field}{String. Name of record id field. Like \code{id}} } +\value{ +List. Log of results for updating metadata +} \description{ Update the structural metadata table } diff --git a/man/fetch_all_json.Rd b/man/fetch_all_json.Rd index 2591cac..5432243 100644 --- a/man/fetch_all_json.Rd +++ b/man/fetch_all_json.Rd @@ -19,3 +19,14 @@ json as string \description{ Get the full outputs of a table as single json object } +\examples{ + +\dontrun{ +base <- "appXXXXXXX" +table_name <- "My Table" + +fetch_all_json(base, table_name) + +} + +} From c67d1c6f7b0701bec1b1ef2cef6b4cd87bcf1a07 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 7 Apr 2023 10:29:43 -0600 Subject: [PATCH 082/126] updated the package namespace --- NAMESPACE | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NAMESPACE b/NAMESPACE index e6db247..7b907d3 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -12,12 +12,13 @@ export(air_dump_to_csv) export(air_dump_to_json) export(air_expand_csv_arrays) export(air_fields_df_template) +export(air_fields_list_from_template) export(air_generate_base_description) export(air_generate_metadata) +export(air_generate_metadata_from_api) export(air_get) export(air_get_attachments) export(air_get_base_description_from_table) -export(air_get_enterprise) export(air_get_json) export(air_get_metadata_from_table) export(air_get_schema) From aae3916a6ae3538ed544762934905a8878494236 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 7 Apr 2023 12:00:21 -0600 Subject: [PATCH 083/126] cleaned up documentation and ran package checks --- DESCRIPTION | 4 +- NAMESPACE | 2 +- R/air_dump.R | 57 +++++++++++++++---- R/air_get_attachments.R | 6 +- R/air_get_schema.R | 35 ------------ R/air_metadata_api.R | 33 +++++++---- R/airtabler.R | 12 +++- man/air_create_description_table.Rd | 6 +- man/air_create_field.Rd | 2 +- man/air_create_metadata_table.Rd | 4 ++ man/air_create_table.Rd | 7 ++- man/air_dump_to_json.Rd | 4 ++ man/air_fields_df_template.Rd | 4 +- man/air_generate_base_description.Rd | 13 ++++- man/air_generate_metadata_from_api.Rd | 10 +++- ...d => air_generate_metadata_from_tables.Rd} | 12 ++-- man/air_get_attachments.Rd | 7 ++- man/air_get_base_description_from_table.Rd | 4 +- man/air_get_metadata_from_table.Rd | 12 +++- man/air_get_schema.Rd | 16 +----- man/air_list_bases.Rd | 6 +- man/air_table_template.Rd | 6 +- man/airtabler-package.Rd | 11 +++- 23 files changed, 174 insertions(+), 99 deletions(-) delete mode 100644 R/air_get_schema.R rename man/{air_generate_metadata.Rd => air_generate_metadata_from_tables.Rd} (82%) diff --git a/DESCRIPTION b/DESCRIPTION index 575601e..338cb32 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -24,7 +24,9 @@ Imports: snakecase, tidyselect, rlang, - stringr + stringr, + tibble, + deposits RoxygenNote: 7.2.1 Suggests: knitr, diff --git a/NAMESPACE b/NAMESPACE index 7b907d3..1817868 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -14,8 +14,8 @@ export(air_expand_csv_arrays) export(air_fields_df_template) export(air_fields_list_from_template) export(air_generate_base_description) -export(air_generate_metadata) export(air_generate_metadata_from_api) +export(air_generate_metadata_from_tables) export(air_get) export(air_get_attachments) export(air_get_base_description_from_table) diff --git a/R/air_dump.R b/R/air_dump.R index 9f3c43d..34e3660 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -39,6 +39,9 @@ set_diff <- function(x,y){ #' Create a new structural metadata table in the base #' +#' @details Structural metadata describes the contents of your base and how they are linked. Structural +#' metadata can largely be derived from the base schema. +#' #' @param base String. Base id #' @param meta_data Data frame. Contains metadata records. From air_generate_metadata* #' @param table_name String. name of the metadata table. default is "Meta Data" @@ -141,6 +144,12 @@ air_create_metadata_table <- function(base,meta_data,table_name = "Meta Data", #' Create the descriptive metadata table for the base #' +#' Descriptive metadata provides information about the base as a whole, who created it, +#' why, when, where can data be accessed, keywords, what license governs data use, etc. +#' Descriptive metadata facilitates data reuse by providing a point of contact for +#' future users, as well as attributes that allow the data to be entered into searchable +#' catalogs or archives. +#' #' @details DCMI terms can be found here \url{https://www.dublincore.org/specifications/dublin-core/dcmi-terms/} #' #' @param base String. Base id @@ -259,7 +268,6 @@ air_create_description_table <- function(base, } -# update metadata table #' Update the structural metadata table #' @@ -374,7 +382,7 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j return(update_log) } -# update description table + #' Update the description table #' #' Update the descriptive metadata table in airtable @@ -518,7 +526,13 @@ air_update_description_table <- function(base,description, table_name = "Descrip } -#' Pull the metadata table from airtable +#' Pull the metadata table from Airtable +#' +#' Airtable allows all users to access the metadata API. +#' The recommended workflow for creating this table is to use +#' air_generate_metadata_from_api to extract the structural metadata from the base +#' schema and then use air_create_metadata_table to add the table to your +#' base. #' #' For information about creating metadata tables in your base see the #' \href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} @@ -533,7 +547,7 @@ air_update_description_table <- function(base,description, table_name = "Descrip #' column are converted to snake_case #' #' @return data.frame with metadata table -#' @export +#' @export air_get_metadata_from_table #' air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, field_names_to_snakecase = TRUE){ # get structural metadata table @@ -571,9 +585,17 @@ air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, f } -# pull data from api and populate metadata table #' Generate structural metadata from the api #' +#' Structural metadata describes the contents of your base and how they are linked. +#' The structural metadata are created from the base schema. The nested schema +#' structure is flattened into a more user-friendly table which can then be +#' inserted as a table into the base with \code{air_created_metadata_table} and/or +#' used in a data export with \code{air_dump}. +#' +#' @details This function requires that the api token has the ability to read +#' the base schema. +#' #' @param base String. Base id #' @param metadata_table_name String. Name of exisiting structural metadata table if it exists #' @param include_metadata_table Logical. Should the structural metadata table be included in the metadata? @@ -674,6 +696,8 @@ air_generate_metadata_from_api <- function(base, #' Generated Metadata from table names #' +#' Deprecated: Use \code{air_generate_metadata_from_api} +#' #' Generates a structural metadata table - the metadata that #' describes how tables and fields fit together. Does not #' include field types. @@ -688,10 +712,10 @@ air_generate_metadata_from_api <- function(base, #' Code runs faster if fewer rows are pulled. #' #' @return data.frame with structural metadata. -#' @export +#' @export air_generate_metadata_from_tables -air_generate_metadata <- function(base, table_names,limit=1){ - warning('For more complete results, use air_generate_metadata_from_api. +air_generate_metadata_from_tables <- function(base, table_names,limit=1){ + warning('Deprecated: For more complete results, use air_generate_metadata_from_api. Airtable does not return fields with empty values - "", false, or [].') meta_data_table <- purrr::map_dfr(table_names,function(x){ table_x <- airtabler::air_get(base,x,limit = limit ) @@ -711,11 +735,12 @@ air_generate_metadata <- function(base, table_names,limit=1){ #' #' Pull a table that has descriptive metadata. #' Requires the following fields: -#' "title","primary_contact","email","base_description" +#' "title","primary_contact","email","description" #' #' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' #' @param table_name String. Name of descriptive metadata table - the metadata that #' describes the base and provides attribution +#' @param field_names_to_snakecase Logical. Should field names be converted to snakecase? #' #' @return data.frame with descriptive metadata. #' @export air_get_base_description_from_table @@ -749,7 +774,12 @@ air_get_base_description_from_table<- function(base, table_name,field_names_to_s #' Generate descriptive metadata #' -#' Creates a data.frame that describes the base. +#' Creates a data.frame that describes the base. Descriptive metadata provides +#' information about the base as a whole: who created it, +#' why, when, where can data be accessed, keywords, what license governs data use, etc. +#' Descriptive metadata facilitates data reuse by providing a point of contact for +#' future users, as well as attributes that allow the data to be entered into searchable +#' catalogs or archives. #' #' @details See \href{https://www.dublincore.org/resources/userguide/creating_metadata/}{dublin core} for inspiration about additional attributes. #' @@ -769,6 +799,7 @@ air_get_base_description_from_table<- function(base, table_name,field_names_to_s #' @param ... String. Additional descriptive metadata elements. See details. #' Additional elements can be added as name pair values e.g. #' \code{ isPartOf = "https://doi.org/00.00000/MyPaper01", isReferencedBy = "https://doi.org/10.48321/MyDMP01"} +#' @param created String. When was the base created? #' #' @return data.frame with descriptive metadata #' @export @@ -778,11 +809,11 @@ air_get_base_description_from_table<- function(base, table_name,field_names_to_s #' air_generate_base_description(title = "My Awesome Base" , #' primary_contact= "Base Creator/Maintainer", #' email = "email@@example.com", -#' base_description = "This base is used to contain my awesome data +#' base_description = "This base contains my awesome data #' from a project studying XXX in YYY. Data in the base were collected #' from 1900-01-01 to 1990-01-01 by researchers at Some Long Term Project.", #' is_part_of = "https://doi.org/10.48321/MyDMP01", -#' is_part_of = "https://doi.org/10.5072/zenodo_sandbox.1062705" +#' isReferencedBy = "https://doi.org/10.5072/zenodo_sandbox.1062705" #' ) #' air_generate_base_description <- function(title = NA, @@ -1127,6 +1158,8 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) #' @param metadata Data.frame.Data frame with structural metadata - describes relationship between tables and fields. #' @param description Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. #' Can be left as NULL if base already contains a table called description +#' @param output_dir String. Where should json files be saved? +#' @param overwrite Logical. If data are not unique, should files be overwritten? #' #' @return List of data.frames. All tables from metadata plus the #' description and metadata tables. diff --git a/R/air_get_attachments.R b/R/air_get_attachments.R index eb5b48c..0b40237 100644 --- a/R/air_get_attachments.R +++ b/R/air_get_attachments.R @@ -1,6 +1,10 @@ #' Get Airtable file attachments #' -#' Get an attachment stored in air tables. For excel files, returns a named list. +#' Extract the contents of an attachment stored in Airtable. Currently only setup +#' to work with Excel files. Planned expansion to other file types. +#' For excel files, returns a named list. +#' +#' @seealso \code{air_download_attachments} #' #' @param base String. ID for the base or app to be fetched #' @param table_name String. Name of the table to be fetched from the base diff --git a/R/air_get_schema.R b/R/air_get_schema.R deleted file mode 100644 index f960bd4..0000000 --- a/R/air_get_schema.R +++ /dev/null @@ -1,35 +0,0 @@ -#' Get base schema -#' -#' Get the schema for the tables in a base. This is a wrapper for the api call -#' Get base schema. -#' -#' @section Using Metadata API: -#' Metadata api is currently available to all users. -#' -#' @param base Airtable base ID -#' @param ... additional paramters -#' -#' @return list of schema -#' @export air_get_schema - -air_get_schema <- function(base,...){ - request_url <- sprintf("%s/%s/tables", air_meta_url, base) - request_url <- utils::URLencode(request_url) - - # call service: - res <- httr::GET( - request_url, - httr::add_headers( - Authorization = paste("Bearer", air_api_key()) - ) - ) - - air_validate(res) - # may need a new air_parse function - - res_content <- httr::content(res,as = "text") - - schema <- jsonlite::fromJSON(res_content) - - return(schema) -} diff --git a/R/air_metadata_api.R b/R/air_metadata_api.R index 2d5f44f..795310a 100644 --- a/R/air_metadata_api.R +++ b/R/air_metadata_api.R @@ -6,13 +6,13 @@ #' @section Using Metadata API: #' Metadata api is currently available to all users. #' -#' @param base Airtable base ID -#' @param ... additional paramters +#' @param base String. Airtable base ID +#' @param ... reserved for additional parameters #' #' @return list of schema #' @export air_get_schema -air_get_schema <- function(base,...){ +air_get_schema <- function(base, ...){ request_url <- sprintf("%s/%s/tables", air_meta_url, base) request_url <- utils::URLencode(request_url) @@ -36,6 +36,7 @@ air_get_schema <- function(base,...){ type_option_map <-function(){ + # will be used to validate options provided to different field types "https://airtable.com/developers/web/api/field-model" # time options are deeply nested @@ -97,7 +98,9 @@ type_option_map <-function(){ #' type = field_types, #' options = field_options) #' -#' table_list <- air_table_template(table_name = "Planet",description = "Planets of Foundation",fields_df = field_tables) +#' table_list <- air_table_template(table_name = "Planet", +#' description = "Planets of Foundation", +#' fields_df = field_tables) #' #' air_create_table(base, table_list) #'} @@ -196,7 +199,7 @@ air_fields_list_from_template <- function(df){ #' #' @examples #' -#' #'\dontrun{ +#' \dontrun{ #' base <- "appQ94sELAtFnXPxx" #' #' base_schema <- air_get_schema(base) @@ -234,7 +237,9 @@ air_fields_list_from_template <- function(df){ #' type = field_types, #' options = field_options) #' -#' table_list <- air_table_template(table_name = "Planet",description = "Planets of Foundation",fields_df = field_tables) +#' table_list <- air_table_template(table_name = "Planet", +#' description = "Planets of Foundation", +#' fields_df = field_tables) #' #' air_create_table(base, table_list) #'} @@ -297,7 +302,7 @@ air_table_template <- function(table_name, description, fields_df ){ #' #' @examples #' -#' #'\dontrun{ +#' \dontrun{ #' base <- "appQ94sELAtFnXPxx" #' #' base_schema <- air_get_schema(base) @@ -335,10 +340,13 @@ air_table_template <- function(table_name, description, fields_df ){ #' type = field_types, #' options = field_options) #' -#' table_list <- air_table_template(table_name = "Planet",description = "Planets of Foundation",fields_df = field_tables) +#' table_list <- air_table_template(table_name = "Planet", +#' description = "Planets of Foundation", +#' fields_df = field_tables) #' #' air_create_table(base, table_list) -#'} +#' +#' } air_create_table <- function(base, table_list){ request_url <- sprintf("%s/%s/tables", air_meta_url, base) request_url <- utils::URLencode(request_url) @@ -380,7 +388,7 @@ air_create_table <- function(base, table_list){ #' @export air_create_field #' #' @examples -#'\donotrun{ +#' \dontrun{ #' base_schema <- air_get_schema(base) #' #' base_schema$tables @@ -445,6 +453,11 @@ air_create_field <- function(base, #' Get list of bases for an Token #' +#' Each token you provisision is given access to a certain set of bases or +#' workspaces. This function lists all bases associated with a token. +#' +#' @param request_url String. URL for api endpoint +#' #' @return list. List of bases a token can access. #' @export air_list_bases #' diff --git a/R/airtabler.R b/R/airtabler.R index 7fcc667..ad22ce8 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -7,8 +7,8 @@ #' and check the API on \url{http://airtable.com/api}. #' #' @section API key: -#' Generate the airtable API key from your Airtable account page -#' (http://airtable.com/account). +#' Generate the Airtable API token from your Airtable account page +#' (http://airtable.com/create/tokens). #' #' \pkg{airtabler} functions will read the API key from #' environment variable \code{AIRTABLE_API_KEY}. To start R session with the @@ -18,6 +18,14 @@ #' \code{AIRTABLE_API_KEY=************} #' #' To check where your R home is, try \code{normalizePath("~")}. +#' +#' The \pkg{usethis} and \pkg{dotenv} packages are useful for setting environment +#' variables. +#' \code{usethis::edit_r_environ} allow you to modify the \code{.Renviron} file. +#' \code{dotenv::load_dot_env} allows you to load environment variables from a +#' \code{.env} file. This second approach is especially helpful if you work +#' with multiple tokens. +#' #' @section Usage: #' Use \code{\link{airtable}} function to get airtable base object #' or just call primitives \code{\link{air_get}}, \code{\link{air_insert}}, diff --git a/man/air_create_description_table.Rd b/man/air_create_description_table.Rd index d194617..5b436ec 100644 --- a/man/air_create_description_table.Rd +++ b/man/air_create_description_table.Rd @@ -30,7 +30,11 @@ air_create_description_table( List. Outputs from creating the table and inserting the records } \description{ -Create the descriptive metadata table for the base +Descriptive metadata provides information about the base as a whole, who created it, +why, when, where can data be accessed, keywords, what license governs data use, etc. +Descriptive metadata facilitates data reuse by providing a point of contact for +future users, as well as attributes that allow the data to be entered into searchable +catalogs or archives. } \details{ DCMI terms can be found here \url{https://www.dublincore.org/specifications/dublin-core/dcmi-terms/} diff --git a/man/air_create_field.Rd b/man/air_create_field.Rd index 3d55165..fb27ab8 100644 --- a/man/air_create_field.Rd +++ b/man/air_create_field.Rd @@ -33,7 +33,7 @@ description of newly created field as a list See https://airtable.com/developers/web/api/create-field } \examples{ -\donotrun{ +\dontrun{ base_schema <- air_get_schema(base) base_schema$tables diff --git a/man/air_create_metadata_table.Rd b/man/air_create_metadata_table.Rd index 2829742..8b8c33d 100644 --- a/man/air_create_metadata_table.Rd +++ b/man/air_create_metadata_table.Rd @@ -32,6 +32,10 @@ List with outcome from creating the table and inserting the records \description{ Create a new structural metadata table in the base } +\details{ +Structural metadata describes the contents of your base and how they are linked. Structural +metadata can largely be derived from the base schema. +} \examples{ \dontrun{ # set base id diff --git a/man/air_create_table.Rd b/man/air_create_table.Rd index 608e4ed..0a604e9 100644 --- a/man/air_create_table.Rd +++ b/man/air_create_table.Rd @@ -23,7 +23,7 @@ See https://airtable.com/developers/web/api/create-table } \examples{ -#'\dontrun{ +\dontrun{ base <- "appQ94sELAtFnXPxx" base_schema <- air_get_schema(base) @@ -61,8 +61,11 @@ field_df<- air_fields_df_template(name = field_names, type = field_types, options = field_options) -table_list <- air_table_template(table_name = "Planet",description = "Planets of Foundation",fields_df = field_tables) +table_list <- air_table_template(table_name = "Planet", + description = "Planets of Foundation", + fields_df = field_tables) air_create_table(base, table_list) + } } diff --git a/man/air_dump_to_json.Rd b/man/air_dump_to_json.Rd index 99814ad..a7a4de7 100644 --- a/man/air_dump_to_json.Rd +++ b/man/air_dump_to_json.Rd @@ -19,6 +19,10 @@ air_dump_to_json( \item{description}{Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. Can be left as NULL if base already contains a table called description} + +\item{output_dir}{String. Where should json files be saved?} + +\item{overwrite}{Logical. If data are not unique, should files be overwritten?} } \value{ List of data.frames. All tables from metadata plus the diff --git a/man/air_fields_df_template.Rd b/man/air_fields_df_template.Rd index 6b522f3..e471c7d 100644 --- a/man/air_fields_df_template.Rd +++ b/man/air_fields_df_template.Rd @@ -62,7 +62,9 @@ field_df<- air_fields_df_template(name = field_names, type = field_types, options = field_options) -table_list <- air_table_template(table_name = "Planet",description = "Planets of Foundation",fields_df = field_tables) +table_list <- air_table_template(table_name = "Planet", + description = "Planets of Foundation", + fields_df = field_tables) air_create_table(base, table_list) } diff --git a/man/air_generate_base_description.Rd b/man/air_generate_base_description.Rd index 703c48c..8686581 100644 --- a/man/air_generate_base_description.Rd +++ b/man/air_generate_base_description.Rd @@ -23,6 +23,8 @@ which a resource is formally known.} \item{creator}{String. Person or people who created the base} +\item{created}{String. When was the base created?} + \item{primary_contact}{String. Person or entity primarily responsible for making the content of a resource} @@ -47,7 +49,12 @@ Additional elements can be added as name pair values e.g. data.frame with descriptive metadata } \description{ -Creates a data.frame that describes the base. +Creates a data.frame that describes the base. Descriptive metadata provides +information about the base as a whole: who created it, +why, when, where can data be accessed, keywords, what license governs data use, etc. +Descriptive metadata facilitates data reuse by providing a point of contact for +future users, as well as attributes that allow the data to be entered into searchable +catalogs or archives. } \details{ See \href{https://www.dublincore.org/resources/userguide/creating_metadata/}{dublin core} for inspiration about additional attributes. @@ -57,11 +64,11 @@ See \href{https://www.dublincore.org/resources/userguide/creating_metadata/}{du air_generate_base_description(title = "My Awesome Base" , primary_contact= "Base Creator/Maintainer", email = "email@example.com", - base_description = "This base is used to contain my awesome data + base_description = "This base contains my awesome data from a project studying XXX in YYY. Data in the base were collected from 1900-01-01 to 1990-01-01 by researchers at Some Long Term Project.", is_part_of = "https://doi.org/10.48321/MyDMP01", - is_part_of = "https://doi.org/10.5072/zenodo_sandbox.1062705" + isReferencedBy = "https://doi.org/10.5072/zenodo_sandbox.1062705" ) } diff --git a/man/air_generate_metadata_from_api.Rd b/man/air_generate_metadata_from_api.Rd index 5692d4f..377dd5e 100644 --- a/man/air_generate_metadata_from_api.Rd +++ b/man/air_generate_metadata_from_api.Rd @@ -21,7 +21,15 @@ air_generate_metadata_from_api( A data frame with metadata } \description{ -Generate structural metadata from the api +Structural metadata describes the contents of your base and how they are linked. +The structural metadata are created from the base schema. The nested schema +structure is flattened into a more user-friendly table which can then be +inserted as a table into the base with \code{air_created_metadata_table} and/or +used in a data export with \code{air_dump}. +} +\details{ +This function requires that the api token has the ability to read +the base schema. } \examples{ diff --git a/man/air_generate_metadata.Rd b/man/air_generate_metadata_from_tables.Rd similarity index 82% rename from man/air_generate_metadata.Rd rename to man/air_generate_metadata_from_tables.Rd index b943357..28576f1 100644 --- a/man/air_generate_metadata.Rd +++ b/man/air_generate_metadata_from_tables.Rd @@ -1,10 +1,10 @@ % Generated by roxygen2: do not edit by hand % Please edit documentation in R/air_dump.R -\name{air_generate_metadata} -\alias{air_generate_metadata} +\name{air_generate_metadata_from_tables} +\alias{air_generate_metadata_from_tables} \title{Generated Metadata from table names} \usage{ -air_generate_metadata(base, table_names, limit = 1) +air_generate_metadata_from_tables(base, table_names, limit = 1) } \arguments{ \item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} @@ -19,11 +19,13 @@ Code runs faster if fewer rows are pulled.} data.frame with structural metadata. } \description{ +Deprecated: Use \code{air_generate_metadata_from_api} +} +\details{ Generates a structural metadata table - the metadata that describes how tables and fields fit together. Does not include field types. -} -\details{ + For information about creating metadata tables in your base see the \href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} } diff --git a/man/air_get_attachments.Rd b/man/air_get_attachments.Rd index 6157f62..ee6f104 100644 --- a/man/air_get_attachments.Rd +++ b/man/air_get_attachments.Rd @@ -44,7 +44,9 @@ Should be one of: excel} named list of data frames } \description{ -Get an attachment stored in air tables. For excel files, returns a named list. +Extract the contents of an attachment stored in Airtable. Currently only setup +to work with Excel files. Planned expansion to other file types. +For excel files, returns a named list. } \examples{ @@ -59,3 +61,6 @@ table_name <- "table with excel attachments" } +\seealso{ +\code{air_download_attachments} +} diff --git a/man/air_get_base_description_from_table.Rd b/man/air_get_base_description_from_table.Rd index 513cf7d..bb26349 100644 --- a/man/air_get_base_description_from_table.Rd +++ b/man/air_get_base_description_from_table.Rd @@ -15,6 +15,8 @@ air_get_base_description_from_table( \item{table_name}{String. Name of descriptive metadata table - the metadata that describes the base and provides attribution} + +\item{field_names_to_snakecase}{Logical. Should field names be converted to snakecase?} } \value{ data.frame with descriptive metadata. @@ -22,7 +24,7 @@ data.frame with descriptive metadata. \description{ Pull a table that has descriptive metadata. Requires the following fields: -"title","primary_contact","email","base_description" +"title","primary_contact","email","description" } \examples{ \dontrun{ diff --git a/man/air_get_metadata_from_table.Rd b/man/air_get_metadata_from_table.Rd index 9d893dc..6b4e442 100644 --- a/man/air_get_metadata_from_table.Rd +++ b/man/air_get_metadata_from_table.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/air_dump.R \name{air_get_metadata_from_table} \alias{air_get_metadata_from_table} -\title{Pull the metadata table from airtable} +\title{Pull the metadata table from Airtable} \usage{ air_get_metadata_from_table( base, @@ -26,9 +26,15 @@ column are converted to snake_case} data.frame with metadata table } \description{ -For information about creating metadata tables in your base see the -\href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} +Airtable allows all users to access the metadata API. +The recommended workflow for creating this table is to use +air_generate_metadata_from_api to extract the structural metadata from the base +schema and then use air_create_metadata_table to add the table to your +base. } \details{ +For information about creating metadata tables in your base see the +\href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} + Requires the following fields: table_name, field_name } diff --git a/man/air_get_schema.Rd b/man/air_get_schema.Rd index 0abb1b4..c11686e 100644 --- a/man/air_get_schema.Rd +++ b/man/air_get_schema.Rd @@ -1,35 +1,25 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/air_get_schema.R, R/air_metadata_api.R +% Please edit documentation in R/air_metadata_api.R \name{air_get_schema} \alias{air_get_schema} \title{Get base schema} \usage{ -air_get_schema(base, ...) - air_get_schema(base, ...) } \arguments{ -\item{base}{Airtable base ID} +\item{base}{String. Airtable base ID} -\item{...}{additional paramters} +\item{...}{reserved for additional parameters} } \value{ -list of schema - list of schema } \description{ -Get the schema for the tables in a base. This is a wrapper for the api call -Get base schema. - Get the schema for the tables in a base. This is a wrapper for the api call Get base schema. } \section{Using Metadata API}{ -Metadata api is currently available to all users. - - Metadata api is currently available to all users. } diff --git a/man/air_list_bases.Rd b/man/air_list_bases.Rd index d907262..842698e 100644 --- a/man/air_list_bases.Rd +++ b/man/air_list_bases.Rd @@ -6,11 +6,15 @@ \usage{ air_list_bases(request_url = "https://api.airtable.com/v0/meta/bases") } +\arguments{ +\item{request_url}{String. URL for api endpoint} +} \value{ list. List of bases a token can access. } \description{ -Get list of bases for an Token +Each token you provisision is given access to a certain set of bases or +workspaces. This function lists all bases associated with a token. } \examples{ diff --git a/man/air_table_template.Rd b/man/air_table_template.Rd index 7bcc057..b5e69df 100644 --- a/man/air_table_template.Rd +++ b/man/air_table_template.Rd @@ -23,7 +23,7 @@ Template for lists that describe tables in Airtable } \examples{ -#'\dontrun{ +\dontrun{ base <- "appQ94sELAtFnXPxx" base_schema <- air_get_schema(base) @@ -61,7 +61,9 @@ field_df<- air_fields_df_template(name = field_names, type = field_types, options = field_options) -table_list <- air_table_template(table_name = "Planet",description = "Planets of Foundation",fields_df = field_tables) +table_list <- air_table_template(table_name = "Planet", + description = "Planets of Foundation", + fields_df = field_tables) air_create_table(base, table_list) } diff --git a/man/airtabler-package.Rd b/man/airtabler-package.Rd index 0e0cb64..3b61112 100644 --- a/man/airtabler-package.Rd +++ b/man/airtabler-package.Rd @@ -16,8 +16,8 @@ Provides access to the Airtable API (\url{http://airtable.com/api}). \section{API key}{ - Generate the airtable API key from your Airtable account page - (http://airtable.com/account). + Generate the Airtable API token from your Airtable account page + (http://airtable.com/create/tokens). \pkg{airtabler} functions will read the API key from environment variable \code{AIRTABLE_API_KEY}. To start R session with the @@ -27,6 +27,13 @@ Provides access to the Airtable API (\url{http://airtable.com/api}). \code{AIRTABLE_API_KEY=************} To check where your R home is, try \code{normalizePath("~")}. + + The \pkg{usethis} and \pkg{dotenv} packages are useful for setting environment + variables. + \code{usethis::edit_r_environ} allow you to modify the \code{.Renviron} file. + \code{dotenv::load_dot_env} allows you to load environment variables from a + \code{.env} file. This second approach is especially helpful if you work + with multiple tokens. } \section{Usage}{ From a92b5fca65a2d472e93974d4c8c70ac7cbbff438 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 11 Apr 2023 09:19:02 -0600 Subject: [PATCH 084/126] adding some things to help with vignette building --- .Rbuildignore | 2 ++ .gitignore | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.Rbuildignore b/.Rbuildignore index 9774842..df5700c 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -6,3 +6,5 @@ ^IMG$ ^readme_cache$ ^\.travis\.yml$ +^doc$ +^Meta$ diff --git a/.gitignore b/.gitignore index dde8cc7..e3f44a1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ readme.html testing_script.R .env inst/doc +/doc/ +/Meta/ From 38da53d2b42817814fcec70cb58674d3ecc65e0a Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 11 Apr 2023 09:44:32 -0600 Subject: [PATCH 085/126] updated version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 338cb32..0711e8f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.2 +Version: 0.2.3 Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes From 30570cc76e6457942cb3fd14094b29be9b13f10c Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 11 Apr 2023 09:45:55 -0600 Subject: [PATCH 086/126] adding inst so that vignettes build --- .gitignore | 1 - inst/doc/Generate-Metadata-and-Backup.R | 172 +++++++ inst/doc/Generate-Metadata-and-Backup.Rmd | 245 +++++++++ inst/doc/Generate-Metadata-and-Backup.html | 560 +++++++++++++++++++++ 4 files changed, 977 insertions(+), 1 deletion(-) create mode 100644 inst/doc/Generate-Metadata-and-Backup.R create mode 100644 inst/doc/Generate-Metadata-and-Backup.Rmd create mode 100644 inst/doc/Generate-Metadata-and-Backup.html diff --git a/.gitignore b/.gitignore index e3f44a1..bd39d79 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,5 @@ readme.html testing_script.R .env -inst/doc /doc/ /Meta/ diff --git a/inst/doc/Generate-Metadata-and-Backup.R b/inst/doc/Generate-Metadata-and-Backup.R new file mode 100644 index 0000000..66cfb3b --- /dev/null +++ b/inst/doc/Generate-Metadata-and-Backup.R @@ -0,0 +1,172 @@ +## ---- include = FALSE--------------------------------------------------------- +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) + +## ----setup-------------------------------------------------------------------- +library(airtabler) + +## ----generate metadata, eval=FALSE-------------------------------------------- +# # set your base id +# base = "appVjIfAo8AJlfTkx" +# ## create a list +# api_metadata <- air_generate_metadata_from_api(base = "appVjIfAo8AJlfTkx") +# str(api_metadata) + +## ----add metadata table, eval=FALSE------------------------------------------ +# status <- air_create_metadata_table(base = base,meta_data = api_metadata) + +## ----descriptive metadata, eval=FALSE----------------------------------------- +# ## create a basic descriptive metadata table +# +# description <- air_generate_base_description(title = "Base Title", +# creator = "Arkady Darell", +# created = "2023-04-03", +# description = "This base is an example for airtabler") +# +# +# # We could have provided keywords as comma separated values but lets make things +# # more interesting by presenting them as a vector +# description_with_keywords <- air_generate_base_description(title = "Base Title", +# creator = "Arkady Darell", +# created = "2023-04-03", +# description = "This base is an example for airtabler", +# keywords = list(c("Example","R","Package","Airtable"))) +# + +## ----add descriptive metadata, eval=FALSE------------------------------------- +# +# ## add our vanilla description table - preferred way to work with description data +# air_create_description_table(base = base,description = description) +# +# +# ## add our description with keywords and - we want keywords to be multiple select +# ## Note this is not the preferred method but it is possible +# +# length(names(description_with_keywords)) +# +# create_choices <- function(x){ +# +# choice_list <- list() +# +# choice_list$choices <- purrr::map(x,function(x_item){ +# list(name = x_item) +# }) +# +# return(list(choice_list)) # wrap in an extra list to +# } +# +# keyword_options <- create_choices(description_with_keywords$keywords[[1]]) +# +# air_create_description_table(base = base,description = description_with_keywords, +# type = c(rep("singleLineText",9),"multipleSelects"), +# options = c(rep(NA,9),keyword_options)) +# +# + +## ----update structural metadata, eval=FALSE----------------------------------- +# +# # get your metadata from the api +# metadata <- air_generate_metadata_from_api(base = base) +# +# # run the function +# update_log <- air_update_metadata_table(base,metadata) +# +# # the update log provides an overview of records that were updated, inserted, or deleted +# # and fields that were created in the event that the structure of your metadata table +# # changed. +# + +## ----update descriptive metadata, eval=FALSE---------------------------------- +# +# base_description <- air_get_base_description_from_table(base = base,table_name = "Description", +# field_names_to_snakecase = FALSE) +# +# base_description$description <- "Keep on updating" +# +# ## since the field types are already established, its a little easier to add multipleSelect keywords +# base_description$keywords[[1]] <- append("New Keyword",base_description$keywords[[1]]) +# +# base_description <- base_description |> +# dplyr::select(-createdTime) +# +# ## if your base_description obj has a record id field, use that for the join. Default is title +# air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id") +# + +## ----dump to R, eval=FALSE---------------------------------------------------- +# +# ## pull down the metadata table from airtable +# metadata <- air_get_metadata_from_table(base = base,table_name = "Meta Data",add_id_field = FALSE) +# +# airtable_base <- air_dump(base = base,metadata = metadata,description = base_description) +# +# summary(airtable_base) +# + +## ----dump to csv, eval=FALSE-------------------------------------------------- +# +# # dump to csv +# air_dump_to_csv(table_list = airtable_base) +# +# # Make a change to the description table to show how hashing works +# base_description <- air_get_base_description_from_table(base = base,table_name = "Description", +# field_names_to_snakecase = FALSE) +# +# ## since the field types are already established, its a little easier to add multipleSelect keywords +# base_description$keywords[[1]] <- append("How will the hash change?",base_description$keywords[[1]]) +# +# base_description <- base_description |> +# dplyr::select(-createdTime) +# +# ## if your base_description obj has a record id field, use that for the join. Default is title +# air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id") +# +# ## pull the changed based down +# airtable_base_updated <- air_dump(base = base,metadata = metadata,description = base_description) +# +# ## dump to csv +# air_dump_to_csv(table_list = airtable_base_updated) + +## ----workspace backup, eval=FALSE--------------------------------------------- +# +# # get all bases associated with token +# bases <- air_list_bases() +# +# # generate the description for our second +# description_2 <- air_generate_base_description(title = bases$bases$name[[2]],creator = "Arkady Darell",created = Sys.Date(),description = "A base to demo bulk backups") +# +# description_log <- air_create_description_table(bases$bases$id[[2]],description = description_2) +# +# metadata_2 <- air_generate_metadata_from_api(bases$bases$id[[2]]) +# +# metadata_log <- air_create_metadata_table(bases$bases$id[[2]],meta_data = metadata_2) +# +# ## add metadata and descriptive data to bases list +# +# base_descriptions <- purrr::map(bases$bases$id,function(base){ +# air_get_base_description_from_table(base,table_name = "Description") +# }) +# +# +# base_metadata <- purrr::map(bases$bases$id,function(base){ +# air_get_metadata_from_table(base,table_name = "Meta Data") +# }) +# +# +# bases$bases$description <- base_descriptions +# bases$bases$metadata <- base_metadata +# +# base_df <- bases$bases +# +# for(i in 1:nrow(base_df)){ +# base_item <- base_df[i,] +# table_list <- air_dump(base = base_item$id ,metadata = base_item$metadata[[1]],description = base_item$description[[1]]) +# +# output_dir <- sprintf("Airtabler Workspace/%s", base_item$name) +# air_dump_to_csv(table_list,output_dir = output_dir) +# } +# +# + diff --git a/inst/doc/Generate-Metadata-and-Backup.Rmd b/inst/doc/Generate-Metadata-and-Backup.Rmd new file mode 100644 index 0000000..672eafb --- /dev/null +++ b/inst/doc/Generate-Metadata-and-Backup.Rmd @@ -0,0 +1,245 @@ +--- +title: "Generate-Metadata-and-Backup" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Generate-Metadata-and-Backup} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(airtabler) +``` + +This vignette will demonstrate how to generate and update metadata for an Airtable +base using the [Airtable metadata api](https://airtable.com/developers/web/api/get-base-schema) then export the base to create an offline backup. + +*Note* Make sure that you have the [environment variable](https://github.com/gaborcsardi/dotenv) `AIRTABLE_API_KEY` set with your [Airtable API token](https://airtable.com/create/tokens). Make sure your token has schema read and write priveleges. + +## Creating metadata from schema + +`air_generate_metadata_from_api` creates a data frame that holds the structural metadata for your base. Structural metadata describes how entities relate to one another, in this case, how fields relate to tables within the base. + + +```{r generate metadata, eval=FALSE} +# set your base id +base = "appVjIfAo8AJlfTkx" +## create a list +api_metadata <- air_generate_metadata_from_api(base = "appVjIfAo8AJlfTkx") +str(api_metadata) +``` + +## Adding a metadata table to the base + +```{r add metadata table, eval=FALSE} +status <- air_create_metadata_table(base = base,meta_data = api_metadata) +``` + +## Create a descriptive metadata table for the base + +The `description` table will describe who created the base, when, and why. +Descriptive metadata is important for long term storage and reuse. The default +terms in the `description` table match core DCMI terms. Additional terms can be +added. + +```{r descriptive metadata, eval=FALSE} +## create a basic descriptive metadata table + +description <- air_generate_base_description(title = "Base Title", + creator = "Arkady Darell", + created = "2023-04-03", + description = "This base is an example for airtabler") + + +# We could have provided keywords as comma separated values but lets make things +# more interesting by presenting them as a vector +description_with_keywords <- air_generate_base_description(title = "Base Title", + creator = "Arkady Darell", + created = "2023-04-03", + description = "This base is an example for airtabler", + keywords = list(c("Example","R","Package","Airtable"))) + +``` + +## Add a description table to the base + +```{r add descriptive metadata, eval=FALSE} + +## add our vanilla description table - preferred way to work with description data +air_create_description_table(base = base,description = description) + + +## add our description with keywords and - we want keywords to be multiple select +## Note this is not the preferred method but it is possible + +length(names(description_with_keywords)) + +create_choices <- function(x){ + + choice_list <- list() + + choice_list$choices <- purrr::map(x,function(x_item){ + list(name = x_item) + }) + + return(list(choice_list)) # wrap in an extra list to +} + +keyword_options <- create_choices(description_with_keywords$keywords[[1]]) + +air_create_description_table(base = base,description = description_with_keywords, + type = c(rep("singleLineText",9),"multipleSelects"), + options = c(rep(NA,9),keyword_options)) + + +``` + +## Updating metadata tables + +Your data will change so your metadata should update as well. + +### Structural Metadata +```{r update structural metadata, eval=FALSE} + +# get your metadata from the api + metadata <- air_generate_metadata_from_api(base = base) + +# run the function +update_log <- air_update_metadata_table(base,metadata) + +# the update log provides an overview of records that were updated, inserted, or deleted +# and fields that were created in the event that the structure of your metadata table +# changed. + +``` + +### Descriptive Metadata + + + +```{r update descriptive metadata, eval=FALSE} + +base_description <- air_get_base_description_from_table(base = base,table_name = "Description", + field_names_to_snakecase = FALSE) + +base_description$description <- "Keep on updating" + +## since the field types are already established, its a little easier to add multipleSelect keywords +base_description$keywords[[1]] <- append("New Keyword",base_description$keywords[[1]]) + +base_description <- base_description |> + dplyr::select(-createdTime) + +## if your base_description obj has a record id field, use that for the join. Default is title +air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id") + +``` + + +## Using the metadata to create a backup + +Now that you have your metadata tables setup, lets see how we can use them to +create backups. The workflow pulls data down from airtable into R where it can +be written to whatever format you like. There is a built in function for writing +to versioned to CSVs. This is the recommended workflow for creating backups. + +### Pull all data into R + +```{r dump to R, eval=FALSE} + +## pull down the metadata table from airtable +metadata <- air_get_metadata_from_table(base = base,table_name = "Meta Data",add_id_field = FALSE) + +airtable_base <- air_dump(base = base,metadata = metadata,description = base_description) + +summary(airtable_base) + +``` + +### Create a set of versioned CSVs + +This function creates a folder with a unique ID based on a hash of the list +passed to table_list. If the data in your base do not change, then the hash +won't change and no new version of the data will be written. + +```{r dump to csv, eval=FALSE} + +# dump to csv +air_dump_to_csv(table_list = airtable_base) + +# Make a change to the description table to show how hashing works +base_description <- air_get_base_description_from_table(base = base,table_name = "Description", + field_names_to_snakecase = FALSE) + +## since the field types are already established, its a little easier to add multipleSelect keywords +base_description$keywords[[1]] <- append("How will the hash change?",base_description$keywords[[1]]) + +base_description <- base_description |> + dplyr::select(-createdTime) + +## if your base_description obj has a record id field, use that for the join. Default is title +air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id") + +## pull the changed based down +airtable_base_updated <- air_dump(base = base,metadata = metadata,description = base_description) + +## dump to csv +air_dump_to_csv(table_list = airtable_base_updated) +``` + +## Backup all bases in a workspace + +*Note* This routine requires personal access tokens that can read and write +data and schema to all bases in a workspace. + +Here we are going to take advantage of the [`list bases`](https://airtable.com/developers/web/api/list-bases) +endpoint in the airtable API. It will list all bases that a token has access to. + +```{r workspace backup, eval=FALSE} + +# get all bases associated with token +bases <- air_list_bases() + +# generate the description for our second +description_2 <- air_generate_base_description(title = bases$bases$name[[2]],creator = "Arkady Darell",created = Sys.Date(),description = "A base to demo bulk backups") + +description_log <- air_create_description_table(bases$bases$id[[2]],description = description_2) + +metadata_2 <- air_generate_metadata_from_api(bases$bases$id[[2]]) + +metadata_log <- air_create_metadata_table(bases$bases$id[[2]],meta_data = metadata_2) + +## add metadata and descriptive data to bases list + +base_descriptions <- purrr::map(bases$bases$id,function(base){ + air_get_base_description_from_table(base,table_name = "Description") +}) + + +base_metadata <- purrr::map(bases$bases$id,function(base){ + air_get_metadata_from_table(base,table_name = "Meta Data") +}) + + +bases$bases$description <- base_descriptions +bases$bases$metadata <- base_metadata + +base_df <- bases$bases + +for(i in 1:nrow(base_df)){ + base_item <- base_df[i,] + table_list <- air_dump(base = base_item$id ,metadata = base_item$metadata[[1]],description = base_item$description[[1]]) + + output_dir <- sprintf("Airtabler Workspace/%s", base_item$name) + air_dump_to_csv(table_list,output_dir = output_dir) +} + + +``` diff --git a/inst/doc/Generate-Metadata-and-Backup.html b/inst/doc/Generate-Metadata-and-Backup.html new file mode 100644 index 0000000..2e4a451 --- /dev/null +++ b/inst/doc/Generate-Metadata-and-Backup.html @@ -0,0 +1,560 @@ + + + + + + + + + + + + + + +Generate-Metadata-and-Backup + + + + + + + + + + + + + + + + + + + + + + + + + + +

Generate-Metadata-and-Backup

+ + + +
library(airtabler)
+

This vignette will demonstrate how to generate and update metadata +for an Airtable base using the Airtable +metadata api then export the base to create an offline backup.

+

Note Make sure that you have the environment variable +AIRTABLE_API_KEY set with your Airtable API token. Make +sure your token has schema read and write priveleges.

+
+

Creating metadata from schema

+

air_generate_metadata_from_api creates a data frame that +holds the structural metadata for your base. Structural metadata +describes how entities relate to one another, in this case, how fields +relate to tables within the base.

+
# set your base id
+base = "appVjIfAo8AJlfTkx"
+## create a list
+api_metadata <- air_generate_metadata_from_api(base = "appVjIfAo8AJlfTkx")
+str(api_metadata)
+
+
+

Adding a metadata table to the base

+
status <- air_create_metadata_table(base = base,meta_data = api_metadata)
+
+
+

Create a descriptive metadata table for the base

+

The description table will describe who created the +base, when, and why. Descriptive metadata is important for long term +storage and reuse. The default terms in the description +table match core DCMI terms. Additional terms can be added.

+
## create a basic descriptive metadata table
+
+description <- air_generate_base_description(title = "Base Title",
+                              creator = "Arkady Darell",
+                              created = "2023-04-03",
+                              description = "This base is an example for airtabler")
+
+
+# We could have provided keywords as comma separated values but lets make things 
+# more interesting by presenting them as a vector
+description_with_keywords <- air_generate_base_description(title = "Base Title",
+                              creator = "Arkady Darell",
+                              created = "2023-04-03",
+                              description = "This base is an example for airtabler",
+                              keywords = list(c("Example","R","Package","Airtable")))
+
+
+

Add a description table to the base

+

+## add our vanilla description table - preferred way to work with description data
+air_create_description_table(base = base,description = description)
+
+
+## add our description with keywords and - we want keywords to be multiple select
+## Note this is not the preferred method but it is possible
+
+length(names(description_with_keywords))
+
+create_choices <- function(x){
+  
+  choice_list <- list()
+  
+   choice_list$choices <- purrr::map(x,function(x_item){
+             list(name = x_item)
+    })
+
+   return(list(choice_list)) # wrap in an extra list to 
+}
+
+keyword_options <- create_choices(description_with_keywords$keywords[[1]])
+
+air_create_description_table(base = base,description = description_with_keywords,
+                             type = c(rep("singleLineText",9),"multipleSelects"),
+                             options = c(rep(NA,9),keyword_options))
+
+
+

Updating metadata tables

+

Your data will change so your metadata should update as well.

+
+

Structural Metadata

+

+# get your metadata from the api
+ metadata <- air_generate_metadata_from_api(base = base)
+
+# run the function
+update_log  <- air_update_metadata_table(base,metadata)
+
+# the update log provides an overview of records that were updated, inserted, or deleted
+# and fields that were created in the event that the structure of your metadata table 
+# changed.
+
+
+

Descriptive Metadata

+

+base_description <- air_get_base_description_from_table(base = base,table_name = "Description",
+                                                        field_names_to_snakecase = FALSE)
+
+base_description$description <- "Keep on updating"
+
+## since the field types are already established, its a little easier to add multipleSelect keywords
+base_description$keywords[[1]] <- append("New Keyword",base_description$keywords[[1]])
+
+base_description <- base_description |> 
+  dplyr::select(-createdTime)
+
+## if your base_description obj has a record id field, use that for the join. Default is title
+air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id")
+
+
+
+

Using the metadata to create a backup

+

Now that you have your metadata tables setup, lets see how we can use +them to create backups. The workflow pulls data down from airtable into +R where it can be written to whatever format you like. There is a built +in function for writing to versioned to CSVs. This is the recommended +workflow for creating backups.

+
+

Pull all data into R

+

+## pull down the metadata table from airtable 
+metadata <- air_get_metadata_from_table(base = base,table_name = "Meta Data",add_id_field = FALSE)
+
+airtable_base <- air_dump(base = base,metadata = metadata,description = base_description)
+
+summary(airtable_base)
+
+
+

Create a set of versioned CSVs

+

This function creates a folder with a unique ID based on a hash of +the list passed to table_list. If the data in your base do not change, +then the hash won’t change and no new version of the data will be +written.

+

+#  dump to csv
+air_dump_to_csv(table_list = airtable_base)
+
+# Make a change to the description table to show how hashing works
+base_description <- air_get_base_description_from_table(base = base,table_name = "Description",
+                                                        field_names_to_snakecase = FALSE)
+
+## since the field types are already established, its a little easier to add multipleSelect keywords
+base_description$keywords[[1]] <- append("How will the hash change?",base_description$keywords[[1]])
+
+base_description <- base_description |> 
+  dplyr::select(-createdTime)
+
+## if your base_description obj has a record id field, use that for the join. Default is title
+air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id")
+
+## pull the changed based down
+airtable_base_updated <- air_dump(base = base,metadata = metadata,description = base_description)
+
+## dump to csv
+air_dump_to_csv(table_list = airtable_base_updated)
+
+
+
+

Backup all bases in a workspace

+

Note This routine requires personal access tokens that can +read and write data and schema to all bases in a workspace.

+

Here we are going to take advantage of the list bases +endpoint in the airtable API. It will list all bases that a token has +access to.

+

+# get all bases associated with token
+bases <- air_list_bases()
+
+# generate the description for our second
+description_2 <- air_generate_base_description(title = bases$bases$name[[2]],creator = "Arkady Darell",created = Sys.Date(),description = "A base to demo bulk backups")
+
+description_log <- air_create_description_table(bases$bases$id[[2]],description = description_2)
+
+metadata_2 <- air_generate_metadata_from_api(bases$bases$id[[2]])
+
+metadata_log <- air_create_metadata_table(bases$bases$id[[2]],meta_data = metadata_2)
+
+## add metadata and descriptive data to bases list
+
+base_descriptions <- purrr::map(bases$bases$id,function(base){
+  air_get_base_description_from_table(base,table_name = "Description")
+})
+
+
+base_metadata <- purrr::map(bases$bases$id,function(base){
+  air_get_metadata_from_table(base,table_name = "Meta Data")
+})
+
+
+bases$bases$description <-  base_descriptions
+bases$bases$metadata <-  base_metadata
+
+base_df <- bases$bases
+
+for(i in 1:nrow(base_df)){
+  base_item <- base_df[i,]
+  table_list <- air_dump(base = base_item$id ,metadata = base_item$metadata[[1]],description = base_item$description[[1]])
+ 
+  output_dir <- sprintf("Airtabler Workspace/%s", base_item$name)
+   air_dump_to_csv(table_list,output_dir = output_dir)
+}
+
+ + + + + + + + + + + From cefbd7590924e5a88335a53569713702b9a99994 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Wed, 12 Apr 2023 15:19:54 -0600 Subject: [PATCH 087/126] made structural and descriptive metadata behave similarly in air_dump --- R/air_dump.R | 34 +- inst/doc/Generate-Metadata-and-Backup.R | 172 ------- inst/doc/Generate-Metadata-and-Backup.Rmd | 245 --------- inst/doc/Generate-Metadata-and-Backup.html | 560 --------------------- man/air_dump.Rd | 5 +- 5 files changed, 34 insertions(+), 982 deletions(-) delete mode 100644 inst/doc/Generate-Metadata-and-Backup.R delete mode 100644 inst/doc/Generate-Metadata-and-Backup.Rmd delete mode 100644 inst/doc/Generate-Metadata-and-Backup.html diff --git a/R/air_dump.R b/R/air_dump.R index 34e3660..5b1dab8 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -844,6 +844,7 @@ air_generate_base_description <- function(title = NA, #' #' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' #' @param metadata Data.frame.Data frame with structural metadata - describes relationship between tables and fields. +#' Can be left as NULL if base already contains a table called meta data. #' @param description Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. #' Can be left as NULL if base already contains a table called description. #' @param add_missing_fields Logical. Should fields described in the metadata data.frame be added to corresponding tables? @@ -862,7 +863,27 @@ air_generate_base_description <- function(title = NA, #' @note To facilitate joining on ids, see purrr::as_vector for converting list type columns to vectors and #' tidyr::unnest for expanding list columns. #' -air_dump <- function(base, metadata, description = NULL, add_missing_fields = TRUE, download_attachments = TRUE, attachment_fields=NULL, field_names_to_snakecase = TRUE,...){ +air_dump <- function(base, metadata= NULL, description = NULL, add_missing_fields = TRUE, download_attachments = TRUE, attachment_fields=NULL, field_names_to_snakecase = TRUE,...){ + + # if metadata is null, check schema for metadata data table, + if(is.null(metadata)){ + message("No metadata provided. Metadata will be retrieved from the metadata + table or generated via API") + #get schema + base_schema <- air_get_schema(base) + # look for meta data table + table_names <- schema$tables$name + + metadata_check <- grepl("meta data",table_names,ignore.case = TRUE) + + if(any(metadata_check)){ + message("Retreiving metadata from table") + metadata <- fetch_all(base = base,table_name = table_names[metadata_check]) + } else { + message("Generating metadata from api with air_generate_metadta_from_api") + metadata <- air_generate_metadata_from_api(base = base) + } + } names(metadata) <- snakecase::to_snake_case(names(metadata)) @@ -888,6 +909,10 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR fields_exp <- metadata[metadata$table_name == x,"field_name"] + if(field_names_to_snakecase){ + fields_exp <- snakecase::to_snake_case(fields_exp) + } + ## pull table - add check for blank tables x_table <- airtabler::fetch_all(base,x) @@ -912,10 +937,13 @@ air_dump <- function(base, metadata, description = NULL, add_missing_fields = TR ignore_fields_pattern <- paste(ignore_fields,collapse = "|") if(length(obs_exp) != 0 & !all(obs_exp %in% ignore_fields)){ missing_fields <- obs_exp[!grepl(ignore_fields_pattern,obs_exp,ignore.case = FALSE)] - missing_fields_glue <- paste(missing_fields, collapse = ", ") + missing_fields_glue <- paste(missing_fields, collapse = "\n") stop(glue::glue('The metadata table is missing the following fields from table {x}: + {missing_fields_glue} - Please update the metadata table.https://airtable.com/{base}')) + + Please update the metadata table via R with air_update_metadata_table + or manually at https://airtable.com/{base}')) } # check for fields in exp and not in obs - append unless frictionless if(add_missing_fields){ diff --git a/inst/doc/Generate-Metadata-and-Backup.R b/inst/doc/Generate-Metadata-and-Backup.R deleted file mode 100644 index 66cfb3b..0000000 --- a/inst/doc/Generate-Metadata-and-Backup.R +++ /dev/null @@ -1,172 +0,0 @@ -## ---- include = FALSE--------------------------------------------------------- -knitr::opts_chunk$set( - collapse = TRUE, - comment = "#>" -) - -## ----setup-------------------------------------------------------------------- -library(airtabler) - -## ----generate metadata, eval=FALSE-------------------------------------------- -# # set your base id -# base = "appVjIfAo8AJlfTkx" -# ## create a list -# api_metadata <- air_generate_metadata_from_api(base = "appVjIfAo8AJlfTkx") -# str(api_metadata) - -## ----add metadata table, eval=FALSE------------------------------------------ -# status <- air_create_metadata_table(base = base,meta_data = api_metadata) - -## ----descriptive metadata, eval=FALSE----------------------------------------- -# ## create a basic descriptive metadata table -# -# description <- air_generate_base_description(title = "Base Title", -# creator = "Arkady Darell", -# created = "2023-04-03", -# description = "This base is an example for airtabler") -# -# -# # We could have provided keywords as comma separated values but lets make things -# # more interesting by presenting them as a vector -# description_with_keywords <- air_generate_base_description(title = "Base Title", -# creator = "Arkady Darell", -# created = "2023-04-03", -# description = "This base is an example for airtabler", -# keywords = list(c("Example","R","Package","Airtable"))) -# - -## ----add descriptive metadata, eval=FALSE------------------------------------- -# -# ## add our vanilla description table - preferred way to work with description data -# air_create_description_table(base = base,description = description) -# -# -# ## add our description with keywords and - we want keywords to be multiple select -# ## Note this is not the preferred method but it is possible -# -# length(names(description_with_keywords)) -# -# create_choices <- function(x){ -# -# choice_list <- list() -# -# choice_list$choices <- purrr::map(x,function(x_item){ -# list(name = x_item) -# }) -# -# return(list(choice_list)) # wrap in an extra list to -# } -# -# keyword_options <- create_choices(description_with_keywords$keywords[[1]]) -# -# air_create_description_table(base = base,description = description_with_keywords, -# type = c(rep("singleLineText",9),"multipleSelects"), -# options = c(rep(NA,9),keyword_options)) -# -# - -## ----update structural metadata, eval=FALSE----------------------------------- -# -# # get your metadata from the api -# metadata <- air_generate_metadata_from_api(base = base) -# -# # run the function -# update_log <- air_update_metadata_table(base,metadata) -# -# # the update log provides an overview of records that were updated, inserted, or deleted -# # and fields that were created in the event that the structure of your metadata table -# # changed. -# - -## ----update descriptive metadata, eval=FALSE---------------------------------- -# -# base_description <- air_get_base_description_from_table(base = base,table_name = "Description", -# field_names_to_snakecase = FALSE) -# -# base_description$description <- "Keep on updating" -# -# ## since the field types are already established, its a little easier to add multipleSelect keywords -# base_description$keywords[[1]] <- append("New Keyword",base_description$keywords[[1]]) -# -# base_description <- base_description |> -# dplyr::select(-createdTime) -# -# ## if your base_description obj has a record id field, use that for the join. Default is title -# air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id") -# - -## ----dump to R, eval=FALSE---------------------------------------------------- -# -# ## pull down the metadata table from airtable -# metadata <- air_get_metadata_from_table(base = base,table_name = "Meta Data",add_id_field = FALSE) -# -# airtable_base <- air_dump(base = base,metadata = metadata,description = base_description) -# -# summary(airtable_base) -# - -## ----dump to csv, eval=FALSE-------------------------------------------------- -# -# # dump to csv -# air_dump_to_csv(table_list = airtable_base) -# -# # Make a change to the description table to show how hashing works -# base_description <- air_get_base_description_from_table(base = base,table_name = "Description", -# field_names_to_snakecase = FALSE) -# -# ## since the field types are already established, its a little easier to add multipleSelect keywords -# base_description$keywords[[1]] <- append("How will the hash change?",base_description$keywords[[1]]) -# -# base_description <- base_description |> -# dplyr::select(-createdTime) -# -# ## if your base_description obj has a record id field, use that for the join. Default is title -# air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id") -# -# ## pull the changed based down -# airtable_base_updated <- air_dump(base = base,metadata = metadata,description = base_description) -# -# ## dump to csv -# air_dump_to_csv(table_list = airtable_base_updated) - -## ----workspace backup, eval=FALSE--------------------------------------------- -# -# # get all bases associated with token -# bases <- air_list_bases() -# -# # generate the description for our second -# description_2 <- air_generate_base_description(title = bases$bases$name[[2]],creator = "Arkady Darell",created = Sys.Date(),description = "A base to demo bulk backups") -# -# description_log <- air_create_description_table(bases$bases$id[[2]],description = description_2) -# -# metadata_2 <- air_generate_metadata_from_api(bases$bases$id[[2]]) -# -# metadata_log <- air_create_metadata_table(bases$bases$id[[2]],meta_data = metadata_2) -# -# ## add metadata and descriptive data to bases list -# -# base_descriptions <- purrr::map(bases$bases$id,function(base){ -# air_get_base_description_from_table(base,table_name = "Description") -# }) -# -# -# base_metadata <- purrr::map(bases$bases$id,function(base){ -# air_get_metadata_from_table(base,table_name = "Meta Data") -# }) -# -# -# bases$bases$description <- base_descriptions -# bases$bases$metadata <- base_metadata -# -# base_df <- bases$bases -# -# for(i in 1:nrow(base_df)){ -# base_item <- base_df[i,] -# table_list <- air_dump(base = base_item$id ,metadata = base_item$metadata[[1]],description = base_item$description[[1]]) -# -# output_dir <- sprintf("Airtabler Workspace/%s", base_item$name) -# air_dump_to_csv(table_list,output_dir = output_dir) -# } -# -# - diff --git a/inst/doc/Generate-Metadata-and-Backup.Rmd b/inst/doc/Generate-Metadata-and-Backup.Rmd deleted file mode 100644 index 672eafb..0000000 --- a/inst/doc/Generate-Metadata-and-Backup.Rmd +++ /dev/null @@ -1,245 +0,0 @@ ---- -title: "Generate-Metadata-and-Backup" -output: rmarkdown::html_vignette -vignette: > - %\VignetteIndexEntry{Generate-Metadata-and-Backup} - %\VignetteEngine{knitr::rmarkdown} - %\VignetteEncoding{UTF-8} ---- - -```{r, include = FALSE} -knitr::opts_chunk$set( - collapse = TRUE, - comment = "#>" -) -``` - -```{r setup} -library(airtabler) -``` - -This vignette will demonstrate how to generate and update metadata for an Airtable -base using the [Airtable metadata api](https://airtable.com/developers/web/api/get-base-schema) then export the base to create an offline backup. - -*Note* Make sure that you have the [environment variable](https://github.com/gaborcsardi/dotenv) `AIRTABLE_API_KEY` set with your [Airtable API token](https://airtable.com/create/tokens). Make sure your token has schema read and write priveleges. - -## Creating metadata from schema - -`air_generate_metadata_from_api` creates a data frame that holds the structural metadata for your base. Structural metadata describes how entities relate to one another, in this case, how fields relate to tables within the base. - - -```{r generate metadata, eval=FALSE} -# set your base id -base = "appVjIfAo8AJlfTkx" -## create a list -api_metadata <- air_generate_metadata_from_api(base = "appVjIfAo8AJlfTkx") -str(api_metadata) -``` - -## Adding a metadata table to the base - -```{r add metadata table, eval=FALSE} -status <- air_create_metadata_table(base = base,meta_data = api_metadata) -``` - -## Create a descriptive metadata table for the base - -The `description` table will describe who created the base, when, and why. -Descriptive metadata is important for long term storage and reuse. The default -terms in the `description` table match core DCMI terms. Additional terms can be -added. - -```{r descriptive metadata, eval=FALSE} -## create a basic descriptive metadata table - -description <- air_generate_base_description(title = "Base Title", - creator = "Arkady Darell", - created = "2023-04-03", - description = "This base is an example for airtabler") - - -# We could have provided keywords as comma separated values but lets make things -# more interesting by presenting them as a vector -description_with_keywords <- air_generate_base_description(title = "Base Title", - creator = "Arkady Darell", - created = "2023-04-03", - description = "This base is an example for airtabler", - keywords = list(c("Example","R","Package","Airtable"))) - -``` - -## Add a description table to the base - -```{r add descriptive metadata, eval=FALSE} - -## add our vanilla description table - preferred way to work with description data -air_create_description_table(base = base,description = description) - - -## add our description with keywords and - we want keywords to be multiple select -## Note this is not the preferred method but it is possible - -length(names(description_with_keywords)) - -create_choices <- function(x){ - - choice_list <- list() - - choice_list$choices <- purrr::map(x,function(x_item){ - list(name = x_item) - }) - - return(list(choice_list)) # wrap in an extra list to -} - -keyword_options <- create_choices(description_with_keywords$keywords[[1]]) - -air_create_description_table(base = base,description = description_with_keywords, - type = c(rep("singleLineText",9),"multipleSelects"), - options = c(rep(NA,9),keyword_options)) - - -``` - -## Updating metadata tables - -Your data will change so your metadata should update as well. - -### Structural Metadata -```{r update structural metadata, eval=FALSE} - -# get your metadata from the api - metadata <- air_generate_metadata_from_api(base = base) - -# run the function -update_log <- air_update_metadata_table(base,metadata) - -# the update log provides an overview of records that were updated, inserted, or deleted -# and fields that were created in the event that the structure of your metadata table -# changed. - -``` - -### Descriptive Metadata - - - -```{r update descriptive metadata, eval=FALSE} - -base_description <- air_get_base_description_from_table(base = base,table_name = "Description", - field_names_to_snakecase = FALSE) - -base_description$description <- "Keep on updating" - -## since the field types are already established, its a little easier to add multipleSelect keywords -base_description$keywords[[1]] <- append("New Keyword",base_description$keywords[[1]]) - -base_description <- base_description |> - dplyr::select(-createdTime) - -## if your base_description obj has a record id field, use that for the join. Default is title -air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id") - -``` - - -## Using the metadata to create a backup - -Now that you have your metadata tables setup, lets see how we can use them to -create backups. The workflow pulls data down from airtable into R where it can -be written to whatever format you like. There is a built in function for writing -to versioned to CSVs. This is the recommended workflow for creating backups. - -### Pull all data into R - -```{r dump to R, eval=FALSE} - -## pull down the metadata table from airtable -metadata <- air_get_metadata_from_table(base = base,table_name = "Meta Data",add_id_field = FALSE) - -airtable_base <- air_dump(base = base,metadata = metadata,description = base_description) - -summary(airtable_base) - -``` - -### Create a set of versioned CSVs - -This function creates a folder with a unique ID based on a hash of the list -passed to table_list. If the data in your base do not change, then the hash -won't change and no new version of the data will be written. - -```{r dump to csv, eval=FALSE} - -# dump to csv -air_dump_to_csv(table_list = airtable_base) - -# Make a change to the description table to show how hashing works -base_description <- air_get_base_description_from_table(base = base,table_name = "Description", - field_names_to_snakecase = FALSE) - -## since the field types are already established, its a little easier to add multipleSelect keywords -base_description$keywords[[1]] <- append("How will the hash change?",base_description$keywords[[1]]) - -base_description <- base_description |> - dplyr::select(-createdTime) - -## if your base_description obj has a record id field, use that for the join. Default is title -air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id") - -## pull the changed based down -airtable_base_updated <- air_dump(base = base,metadata = metadata,description = base_description) - -## dump to csv -air_dump_to_csv(table_list = airtable_base_updated) -``` - -## Backup all bases in a workspace - -*Note* This routine requires personal access tokens that can read and write -data and schema to all bases in a workspace. - -Here we are going to take advantage of the [`list bases`](https://airtable.com/developers/web/api/list-bases) -endpoint in the airtable API. It will list all bases that a token has access to. - -```{r workspace backup, eval=FALSE} - -# get all bases associated with token -bases <- air_list_bases() - -# generate the description for our second -description_2 <- air_generate_base_description(title = bases$bases$name[[2]],creator = "Arkady Darell",created = Sys.Date(),description = "A base to demo bulk backups") - -description_log <- air_create_description_table(bases$bases$id[[2]],description = description_2) - -metadata_2 <- air_generate_metadata_from_api(bases$bases$id[[2]]) - -metadata_log <- air_create_metadata_table(bases$bases$id[[2]],meta_data = metadata_2) - -## add metadata and descriptive data to bases list - -base_descriptions <- purrr::map(bases$bases$id,function(base){ - air_get_base_description_from_table(base,table_name = "Description") -}) - - -base_metadata <- purrr::map(bases$bases$id,function(base){ - air_get_metadata_from_table(base,table_name = "Meta Data") -}) - - -bases$bases$description <- base_descriptions -bases$bases$metadata <- base_metadata - -base_df <- bases$bases - -for(i in 1:nrow(base_df)){ - base_item <- base_df[i,] - table_list <- air_dump(base = base_item$id ,metadata = base_item$metadata[[1]],description = base_item$description[[1]]) - - output_dir <- sprintf("Airtabler Workspace/%s", base_item$name) - air_dump_to_csv(table_list,output_dir = output_dir) -} - - -``` diff --git a/inst/doc/Generate-Metadata-and-Backup.html b/inst/doc/Generate-Metadata-and-Backup.html deleted file mode 100644 index 2e4a451..0000000 --- a/inst/doc/Generate-Metadata-and-Backup.html +++ /dev/null @@ -1,560 +0,0 @@ - - - - - - - - - - - - - - -Generate-Metadata-and-Backup - - - - - - - - - - - - - - - - - - - - - - - - - - -

Generate-Metadata-and-Backup

- - - -
library(airtabler)
-

This vignette will demonstrate how to generate and update metadata -for an Airtable base using the Airtable -metadata api then export the base to create an offline backup.

-

Note Make sure that you have the environment variable -AIRTABLE_API_KEY set with your Airtable API token. Make -sure your token has schema read and write priveleges.

-
-

Creating metadata from schema

-

air_generate_metadata_from_api creates a data frame that -holds the structural metadata for your base. Structural metadata -describes how entities relate to one another, in this case, how fields -relate to tables within the base.

-
# set your base id
-base = "appVjIfAo8AJlfTkx"
-## create a list
-api_metadata <- air_generate_metadata_from_api(base = "appVjIfAo8AJlfTkx")
-str(api_metadata)
-
-
-

Adding a metadata table to the base

-
status <- air_create_metadata_table(base = base,meta_data = api_metadata)
-
-
-

Create a descriptive metadata table for the base

-

The description table will describe who created the -base, when, and why. Descriptive metadata is important for long term -storage and reuse. The default terms in the description -table match core DCMI terms. Additional terms can be added.

-
## create a basic descriptive metadata table
-
-description <- air_generate_base_description(title = "Base Title",
-                              creator = "Arkady Darell",
-                              created = "2023-04-03",
-                              description = "This base is an example for airtabler")
-
-
-# We could have provided keywords as comma separated values but lets make things 
-# more interesting by presenting them as a vector
-description_with_keywords <- air_generate_base_description(title = "Base Title",
-                              creator = "Arkady Darell",
-                              created = "2023-04-03",
-                              description = "This base is an example for airtabler",
-                              keywords = list(c("Example","R","Package","Airtable")))
-
-
-

Add a description table to the base

-

-## add our vanilla description table - preferred way to work with description data
-air_create_description_table(base = base,description = description)
-
-
-## add our description with keywords and - we want keywords to be multiple select
-## Note this is not the preferred method but it is possible
-
-length(names(description_with_keywords))
-
-create_choices <- function(x){
-  
-  choice_list <- list()
-  
-   choice_list$choices <- purrr::map(x,function(x_item){
-             list(name = x_item)
-    })
-
-   return(list(choice_list)) # wrap in an extra list to 
-}
-
-keyword_options <- create_choices(description_with_keywords$keywords[[1]])
-
-air_create_description_table(base = base,description = description_with_keywords,
-                             type = c(rep("singleLineText",9),"multipleSelects"),
-                             options = c(rep(NA,9),keyword_options))
-
-
-

Updating metadata tables

-

Your data will change so your metadata should update as well.

-
-

Structural Metadata

-

-# get your metadata from the api
- metadata <- air_generate_metadata_from_api(base = base)
-
-# run the function
-update_log  <- air_update_metadata_table(base,metadata)
-
-# the update log provides an overview of records that were updated, inserted, or deleted
-# and fields that were created in the event that the structure of your metadata table 
-# changed.
-
-
-

Descriptive Metadata

-

-base_description <- air_get_base_description_from_table(base = base,table_name = "Description",
-                                                        field_names_to_snakecase = FALSE)
-
-base_description$description <- "Keep on updating"
-
-## since the field types are already established, its a little easier to add multipleSelect keywords
-base_description$keywords[[1]] <- append("New Keyword",base_description$keywords[[1]])
-
-base_description <- base_description |> 
-  dplyr::select(-createdTime)
-
-## if your base_description obj has a record id field, use that for the join. Default is title
-air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id")
-
-
-
-

Using the metadata to create a backup

-

Now that you have your metadata tables setup, lets see how we can use -them to create backups. The workflow pulls data down from airtable into -R where it can be written to whatever format you like. There is a built -in function for writing to versioned to CSVs. This is the recommended -workflow for creating backups.

-
-

Pull all data into R

-

-## pull down the metadata table from airtable 
-metadata <- air_get_metadata_from_table(base = base,table_name = "Meta Data",add_id_field = FALSE)
-
-airtable_base <- air_dump(base = base,metadata = metadata,description = base_description)
-
-summary(airtable_base)
-
-
-

Create a set of versioned CSVs

-

This function creates a folder with a unique ID based on a hash of -the list passed to table_list. If the data in your base do not change, -then the hash won’t change and no new version of the data will be -written.

-

-#  dump to csv
-air_dump_to_csv(table_list = airtable_base)
-
-# Make a change to the description table to show how hashing works
-base_description <- air_get_base_description_from_table(base = base,table_name = "Description",
-                                                        field_names_to_snakecase = FALSE)
-
-## since the field types are already established, its a little easier to add multipleSelect keywords
-base_description$keywords[[1]] <- append("How will the hash change?",base_description$keywords[[1]])
-
-base_description <- base_description |> 
-  dplyr::select(-createdTime)
-
-## if your base_description obj has a record id field, use that for the join. Default is title
-air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id")
-
-## pull the changed based down
-airtable_base_updated <- air_dump(base = base,metadata = metadata,description = base_description)
-
-## dump to csv
-air_dump_to_csv(table_list = airtable_base_updated)
-
-
-
-

Backup all bases in a workspace

-

Note This routine requires personal access tokens that can -read and write data and schema to all bases in a workspace.

-

Here we are going to take advantage of the list bases -endpoint in the airtable API. It will list all bases that a token has -access to.

-

-# get all bases associated with token
-bases <- air_list_bases()
-
-# generate the description for our second
-description_2 <- air_generate_base_description(title = bases$bases$name[[2]],creator = "Arkady Darell",created = Sys.Date(),description = "A base to demo bulk backups")
-
-description_log <- air_create_description_table(bases$bases$id[[2]],description = description_2)
-
-metadata_2 <- air_generate_metadata_from_api(bases$bases$id[[2]])
-
-metadata_log <- air_create_metadata_table(bases$bases$id[[2]],meta_data = metadata_2)
-
-## add metadata and descriptive data to bases list
-
-base_descriptions <- purrr::map(bases$bases$id,function(base){
-  air_get_base_description_from_table(base,table_name = "Description")
-})
-
-
-base_metadata <- purrr::map(bases$bases$id,function(base){
-  air_get_metadata_from_table(base,table_name = "Meta Data")
-})
-
-
-bases$bases$description <-  base_descriptions
-bases$bases$metadata <-  base_metadata
-
-base_df <- bases$bases
-
-for(i in 1:nrow(base_df)){
-  base_item <- base_df[i,]
-  table_list <- air_dump(base = base_item$id ,metadata = base_item$metadata[[1]],description = base_item$description[[1]])
- 
-  output_dir <- sprintf("Airtabler Workspace/%s", base_item$name)
-   air_dump_to_csv(table_list,output_dir = output_dir)
-}
-
- - - - - - - - - - - diff --git a/man/air_dump.Rd b/man/air_dump.Rd index 590abd5..e22b473 100644 --- a/man/air_dump.Rd +++ b/man/air_dump.Rd @@ -6,7 +6,7 @@ \usage{ air_dump( base, - metadata, + metadata = NULL, description = NULL, add_missing_fields = TRUE, download_attachments = TRUE, @@ -18,7 +18,8 @@ air_dump( \arguments{ \item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} -\item{metadata}{Data.frame.Data frame with structural metadata - describes relationship between tables and fields.} +\item{metadata}{Data.frame.Data frame with structural metadata - describes relationship between tables and fields. +Can be left as NULL if base already contains a table called meta data.} \item{description}{Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. Can be left as NULL if base already contains a table called description.} From 11f5fb21741c4cf36ece448a8eb92f53c46422f7 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Wed, 12 Apr 2023 15:34:20 -0600 Subject: [PATCH 088/126] Update DESCRIPTION --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 0711e8f..b1afbef 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.3 +Version: 0.2.5 Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes From 8e39c615df01c5fbc5d221046ba11a88e89f0e77 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 10:02:59 -0600 Subject: [PATCH 089/126] adding a snake_case option --- R/air_dump.R | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/R/air_dump.R b/R/air_dump.R index 5b1dab8..0a19a41 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -599,6 +599,7 @@ air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, f #' @param base String. Base id #' @param metadata_table_name String. Name of exisiting structural metadata table if it exists #' @param include_metadata_table Logical. Should the structural metadata table be included in the metadata? +#' @param field_names_to_snake_case Logical. Should the field names in the metadata table be snake_case? #' #' @return A data frame with metadata #' @export air_generate_metadata_from_api @@ -614,7 +615,8 @@ air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, f air_generate_metadata_from_api <- function(base, metadata_table_name = "Meta Data", - include_metadata_table = FALSE){ + include_metadata_table = FALSE, + field_names_to_snake_case = TRUE){ # get base schema schema <- air_get_schema(base) @@ -686,6 +688,11 @@ air_generate_metadata_from_api <- function(base, primary_key = as.character(x$primaryFieldId == fields_df$id) ) + if(!field_names_to_snake_case){ + names(md_df) <- c("Field Name","Table Name", "Field Desc","Field Type", + "Field ID", "Table ID", "Field Opts", "Primary Key") + } + }) From d39072d8b71e02d64c4c4030c3386927a0b2b285 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 10:11:31 -0600 Subject: [PATCH 090/126] updating docs --- man/air_generate_metadata_from_api.Rd | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/man/air_generate_metadata_from_api.Rd b/man/air_generate_metadata_from_api.Rd index 377dd5e..bccfda2 100644 --- a/man/air_generate_metadata_from_api.Rd +++ b/man/air_generate_metadata_from_api.Rd @@ -7,7 +7,8 @@ air_generate_metadata_from_api( base, metadata_table_name = "Meta Data", - include_metadata_table = FALSE + include_metadata_table = FALSE, + field_names_to_snake_case = TRUE ) } \arguments{ @@ -16,6 +17,8 @@ air_generate_metadata_from_api( \item{metadata_table_name}{String. Name of exisiting structural metadata table if it exists} \item{include_metadata_table}{Logical. Should the structural metadata table be included in the metadata?} + +\item{field_names_to_snake_case}{Logical. Should the field names in the metadata table be snake_case?} } \value{ A data frame with metadata From 6e3924dc7ae33248a61e834a6506ec859c2d9709 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 10:12:26 -0600 Subject: [PATCH 091/126] updating version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index b1afbef..284e349 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.5 +Version: 0.2.6 Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes From ad3a889722fb375a2afe61ff0482a76f9b17be74 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 10:18:14 -0600 Subject: [PATCH 092/126] explicitly return df in metadata from api --- R/air_dump.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/air_dump.R b/R/air_dump.R index 0a19a41..6a416a3 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -693,7 +693,7 @@ air_generate_metadata_from_api <- function(base, "Field ID", "Table ID", "Field Opts", "Primary Key") } - + return(md_df) }) return(metadata_df) From c3597a12a83f0c094ee28c47c8e67b7fa3d3ce83 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 10:18:40 -0600 Subject: [PATCH 093/126] updated version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 284e349..5642376 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.6 +Version: 0.2.7 Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes From defa30afbad0876e46b9ebdc1ea5d867278ba5db Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 10:41:47 -0600 Subject: [PATCH 094/126] added snake_case rules for getting metadta from table --- R/air_dump.R | 12 ++++++++++-- man/air_get_metadata_from_table.Rd | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index 6a416a3..6e915b0 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -544,7 +544,7 @@ air_update_description_table <- function(base,description, table_name = "Descrip #' describes how tables and fields fit together. #' @param add_id_field Logical. If true, an "id" field is added to each table #' @param field_names_to_snakecase Logical. If true, values in the field_names -#' column are converted to snake_case +#' column and the field in the metadata table themselves are are converted to snake_case #' #' @return data.frame with metadata table #' @export air_get_metadata_from_table @@ -552,8 +552,12 @@ air_update_description_table <- function(base,description, table_name = "Descrip air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, field_names_to_snakecase = TRUE){ # get structural metadata table str_metadata <- airtabler::fetch_all(base,table_name) + + # get original table names + str_md_names <- names(str_metadata) + ## check for table_name, field_name - names(str_metadata) <- snakecase::to_snake_case(names(str_metadata)) + names(str_metadata) <- snakecase::to_snake_case(str_md_names) required_fields <- c("table_name","field_name") if(!all(required_fields %in% names(str_metadata))){ @@ -581,6 +585,10 @@ air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, f } + if(!field_names_to_snakecase){ + names(str_metadata) <- str_md_names + } + return(str_metadata) } diff --git a/man/air_get_metadata_from_table.Rd b/man/air_get_metadata_from_table.Rd index 6b4e442..d365859 100644 --- a/man/air_get_metadata_from_table.Rd +++ b/man/air_get_metadata_from_table.Rd @@ -20,7 +20,7 @@ describes how tables and fields fit together.} \item{add_id_field}{Logical. If true, an "id" field is added to each table} \item{field_names_to_snakecase}{Logical. If true, values in the field_names -column are converted to snake_case} +column and the field in the metadata table themselves are are converted to snake_case} } \value{ data.frame with metadata table From ca839ed90a60cc0412408441734c867950583951 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 11:26:26 -0600 Subject: [PATCH 095/126] added status updates --- R/air_dump.R | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index 6e915b0..34a4f84 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -311,6 +311,7 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j #create any new fields from meta_data + message("creating log") update_log <- list( fields_created = NA, records_updated = NA, @@ -318,6 +319,7 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j records_deleted = NA ) + message("checking if any fields need to be added") col_check <- !names(meta_data) %in% names(current_metadata_table) if(all(col_check)){ @@ -327,6 +329,7 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j } if(any(col_check)){ + message("creating missing fields") cols_to_create <- names(meta_data)[col_check] fields_created <- air_create_field(base = base, @@ -341,7 +344,7 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j # compare with updated values ## use field ids - + message("checking which fields need to be updated") ## assumes a certain structure for metadata min_update_df <- current_metadata_table[,c(join_field,record_id_field)] @@ -349,13 +352,13 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j records_to_update <- dplyr::inner_join(meta_data,min_update_df,by = join_field ) # update records - + message("updating records") records_updated <- air_update_data_frame(base, table_name, records_to_update$id,records_to_update) update_log$records_updated <- records_updated # insert new records - + message("added new records") records_to_insert <- dplyr::anti_join(meta_data,min_update_df,by = join_field) records_inserted <- air_insert_data_frame(base, table_name,records_to_insert) @@ -364,6 +367,7 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j # drop records no longer in meta data + message("deleting records do longer in the base") records_to_delete <- dplyr::anti_join(min_update_df,meta_data,by = join_field) From 9ee6d86e37dfd7e20edfa29ec81a6edb05e44b04 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 12:07:30 -0600 Subject: [PATCH 096/126] updated messages --- R/air_dump.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/air_dump.R b/R/air_dump.R index 34a4f84..7cd0f97 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -367,16 +367,18 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j # drop records no longer in meta data - message("deleting records do longer in the base") + message("Checking for records no longer in the base") records_to_delete <- dplyr::anti_join(min_update_df,meta_data,by = join_field) if(nrow(records_to_delete) >0){ + message("Deleting records no longer in the base") records_deleted <- purrr::map(records_to_delete$id, function(id){ air_delete(base, table_name,id) }) } else { + message("No Records deleted") records_deleted <- "No records deleted" } From ba5801e750805ad19c88061a6cd0ecc2e62fb77c Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 12:41:40 -0600 Subject: [PATCH 097/126] added a polite option for downloads --- R/air_dump.R | 20 +++++++++++++++++--- man/air_dump.Rd | 7 ++++++- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index 7cd0f97..4707213 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -876,15 +876,24 @@ air_generate_base_description <- function(title = NA, #' with type multipleAttachments in metadata. #' @param field_names_to_snakecase Logical. Should field names be #' converted to snake case? +#' @param polite_downloads Logical. Use if downloading many files. Sets a delay +#' so that server is not overwhelmed by requests. #' #' @return List of data.frames. All tables from metadata plus the #' description and metadata tables. #' @export air_dump #' -#' @note To facilitate joining on ids, see purrr::as_vector for converting list type columns to vectors and +#' @note To facilitate joining on ids, see purrr::as_vector for converting list +#' type columns to vectors and #' tidyr::unnest for expanding list columns. #' -air_dump <- function(base, metadata= NULL, description = NULL, add_missing_fields = TRUE, download_attachments = TRUE, attachment_fields=NULL, field_names_to_snakecase = TRUE,...){ +air_dump <- function(base, metadata= NULL, description = NULL, + add_missing_fields = TRUE, + download_attachments = TRUE, + attachment_fields=NULL, + polite_downloads = TRUE, + field_names_to_snakecase = TRUE, + ...){ # if metadata is null, check schema for metadata data table, if(is.null(metadata)){ @@ -893,7 +902,7 @@ air_dump <- function(base, metadata= NULL, description = NULL, add_missing_field #get schema base_schema <- air_get_schema(base) # look for meta data table - table_names <- schema$tables$name + table_names <- base_schema$tables$name metadata_check <- grepl("meta data",table_names,ignore.case = TRUE) @@ -1006,7 +1015,12 @@ air_dump <- function(base, metadata= NULL, description = NULL, add_missing_field } ## build up attachment fields on x_table + sleep_time <- 0 + if(polite_downloads){ + sleep_time <- 0.01 + } for(af in attachment_fields){ + Sys.sleep(sleep_time) x_table <- air_download_attachments(x_table,field = af,...) } } diff --git a/man/air_dump.Rd b/man/air_dump.Rd index e22b473..e0ed61e 100644 --- a/man/air_dump.Rd +++ b/man/air_dump.Rd @@ -11,6 +11,7 @@ air_dump( add_missing_fields = TRUE, download_attachments = TRUE, attachment_fields = NULL, + polite_downloads = TRUE, field_names_to_snakecase = TRUE, ... ) @@ -32,6 +33,9 @@ Can be left as NULL if base already contains a table called description.} What field(s) should files be downloaded from? Default is to download all fields with type multipleAttachments in metadata.} +\item{polite_downloads}{Logical. Use if downloading many files. Sets a delay +so that server is not overwhelmed by requests.} + \item{field_names_to_snakecase}{Logical. Should field names be converted to snake case?} @@ -45,6 +49,7 @@ description and metadata tables. Dump all tables from a base into R } \note{ -To facilitate joining on ids, see purrr::as_vector for converting list type columns to vectors and +To facilitate joining on ids, see purrr::as_vector for converting list + type columns to vectors and tidyr::unnest for expanding list columns. } From 3f28e9d26c3cf019cfa58571cf2d8ca21f096574 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 12:42:30 -0600 Subject: [PATCH 098/126] updated version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 5642376..793f9f5 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.7 +Version: 0.2.8 Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes From 5482298899ebf046570e70417ac2279b506f829f Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 15:43:49 -0600 Subject: [PATCH 099/126] minor changes to default and print behavior --- R/air_download_attachments.R | 2 +- R/air_dump.R | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/air_download_attachments.R b/R/air_download_attachments.R index cadfbbe..f87c7b7 100644 --- a/R/air_download_attachments.R +++ b/R/air_download_attachments.R @@ -83,7 +83,7 @@ air_download_attachments <- function(x, field, dir_name = "downloads",...){ dest <- sprintf("%s/%s_%s", dir_name,x$id,x$filename) a <- utils::download.file(url = x$url,destfile = dest) - print(a) + #print(a) return(dest) }) diff --git a/R/air_dump.R b/R/air_dump.R index 4707213..5c3669e 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -555,7 +555,7 @@ air_update_description_table <- function(base,description, table_name = "Descrip #' @return data.frame with metadata table #' @export air_get_metadata_from_table #' -air_get_metadata_from_table <- function(base, table_name, add_id_field = TRUE, field_names_to_snakecase = TRUE){ +air_get_metadata_from_table <- function(base, table_name, add_id_field = FALSE, field_names_to_snakecase = TRUE){ # get structural metadata table str_metadata <- airtabler::fetch_all(base,table_name) From c7136f82a1b9f8bd65fc6dff448ac1b3f1373f36 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 16:24:23 -0600 Subject: [PATCH 100/126] add check for attachments that already exist locally --- R/air_download_attachments.R | 15 ++++++++++++++- man/air_get_metadata_from_table.Rd | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/R/air_download_attachments.R b/R/air_download_attachments.R index f87c7b7..e616a74 100644 --- a/R/air_download_attachments.R +++ b/R/air_download_attachments.R @@ -82,8 +82,21 @@ air_download_attachments <- function(x, field, dir_name = "downloads",...){ dest <- sprintf("%s/%s_%s", dir_name,x$id,x$filename) + # sometimes the same file is attached multiple times + # if the file is already downloaded, don't add it again + # each attachment gets a unique id, so if the file changes, + # that id changes + + + if(all(file.exists(dest))){ + + not_downloaded_message <- glue::glue("\nFile already exists, not downloaded\n{dest}\n") + print(not_downloaded_message) + return(dest) + } + a <- utils::download.file(url = x$url,destfile = dest) - #print(a) + print(a) return(dest) }) diff --git a/man/air_get_metadata_from_table.Rd b/man/air_get_metadata_from_table.Rd index d365859..10498d4 100644 --- a/man/air_get_metadata_from_table.Rd +++ b/man/air_get_metadata_from_table.Rd @@ -7,7 +7,7 @@ air_get_metadata_from_table( base, table_name, - add_id_field = TRUE, + add_id_field = FALSE, field_names_to_snakecase = TRUE ) } From 68a79f62022bd846ebf392ab211a3ed30ec3ac91 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 13 Apr 2023 18:07:41 -0600 Subject: [PATCH 101/126] added option to copy attachments into csv dump folder --- R/air_dump.R | 8 +++++++- man/air_dump_to_csv.Rd | 9 ++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index 5c3669e..80c1432 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -1157,10 +1157,11 @@ flatten_col_to_chr <- function(data_frame){ #' @param table_list List. List of data.frames output from \code{air_dump} #' @param output_dir String. Folder containing output files #' @param overwrite Logical. Should outputs be overwritten if they already exist? +#' @param attachments_dir String. What folder are base attachments stored in? #' #' @return Vector of file paths #' @export -air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE){ +air_dump_to_csv <- function(table_list,output_dir= "outputs",attachments_dir=NULL, overwrite = FALSE){ # create a unique id for the data output_id <- rlang::hash(table_list) @@ -1205,6 +1206,11 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs", overwrite = FALSE) file.copy(from = outputs_list,to = output_dir_path_final,recursive = FALSE ,copy.mode = TRUE) + ## copy attachments into folder + if(!is.null(attachments_dir)){ + message("copying attachments") + file.copy(from = attachments_dir, to = output_dir_path_final,recursive = TRUE ,copy.mode = TRUE ) + } return(list.files(output_dir_path_final,full.names = TRUE)) } diff --git a/man/air_dump_to_csv.Rd b/man/air_dump_to_csv.Rd index 37cf2e1..4a916ae 100644 --- a/man/air_dump_to_csv.Rd +++ b/man/air_dump_to_csv.Rd @@ -4,13 +4,20 @@ \alias{air_dump_to_csv} \title{Save air_dump output to csv} \usage{ -air_dump_to_csv(table_list, output_dir = "outputs", overwrite = FALSE) +air_dump_to_csv( + table_list, + output_dir = "outputs", + attachments_dir = NULL, + overwrite = FALSE +) } \arguments{ \item{table_list}{List. List of data.frames output from \code{air_dump}} \item{output_dir}{String. Folder containing output files} +\item{attachments_dir}{String. What folder are base attachments stored in?} + \item{overwrite}{Logical. Should outputs be overwritten if they already exist?} } \value{ From 9afc0b84285645b73d9bd9dad4e76999068df6db Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 14 Apr 2023 15:20:43 -0600 Subject: [PATCH 102/126] wrapped download in map so url is length 1 character vector --- R/air_download_attachments.R | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/R/air_download_attachments.R b/R/air_download_attachments.R index e616a74..5a6d169 100644 --- a/R/air_download_attachments.R +++ b/R/air_download_attachments.R @@ -95,8 +95,13 @@ air_download_attachments <- function(x, field, dir_name = "downloads",...){ return(dest) } - a <- utils::download.file(url = x$url,destfile = dest) - print(a) + # wrap in a map so that it works on linux systems where urls must explicitly + # be a length one character vector + purrr::map2(x$url, dest, function(url_item, dest_item){ + a <- utils::download.file(url = url_item,destfile = dest_item) + print(a) + }) + return(dest) }) From 4933005fa3460817f5a32b5002f5170b1feb8d7f Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 14 Apr 2023 15:28:37 -0600 Subject: [PATCH 103/126] added explicit check for character and lenght requirements --- R/air_download_attachments.R | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/R/air_download_attachments.R b/R/air_download_attachments.R index 5a6d169..f7ec90f 100644 --- a/R/air_download_attachments.R +++ b/R/air_download_attachments.R @@ -98,8 +98,13 @@ air_download_attachments <- function(x, field, dir_name = "downloads",...){ # wrap in a map so that it works on linux systems where urls must explicitly # be a length one character vector purrr::map2(x$url, dest, function(url_item, dest_item){ + if(is.character(url_item)& length(url_item )==1){ a <- utils::download.file(url = url_item,destfile = dest_item) print(a) + } else { + print("url item not character or length greater than one") + print(url_item) + } }) From b2f2798b2a9aa7b00391d31e2132e992e52522eb Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 25 Apr 2023 15:37:10 -0600 Subject: [PATCH 104/126] update_metadata table can now handle empty views --- DESCRIPTION | 4 ++-- R/air_dump.R | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 793f9f5..a17b735 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.8 +Version: 0.2.9 Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes @@ -27,7 +27,7 @@ Imports: stringr, tibble, deposits -RoxygenNote: 7.2.1 +RoxygenNote: 7.2.3 Suggests: knitr, rmarkdown diff --git a/R/air_dump.R b/R/air_dump.R index 80c1432..204be4a 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -319,6 +319,18 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j records_deleted = NA ) + # if the current metadata table is empty, then insert records + if(is.character(current_metadata_table)){ + message("added new records") + records_to_insert <- dplyr::anti_join(meta_data,min_update_df,by = join_field) + + records_inserted <- air_insert_data_frame(base, table_name,records_to_insert) + + update_log$records_inserted <- records_inserted + + return(update_log) + } + message("checking if any fields need to be added") col_check <- !names(meta_data) %in% names(current_metadata_table) From d807c7d9acf9b00ddc8857bafb5f43b192aca4be Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 25 Apr 2023 15:45:29 -0600 Subject: [PATCH 105/126] dropping filtering join for empty views --- R/air_dump.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/air_dump.R b/R/air_dump.R index 204be4a..1cfe4cd 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -322,7 +322,7 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j # if the current metadata table is empty, then insert records if(is.character(current_metadata_table)){ message("added new records") - records_to_insert <- dplyr::anti_join(meta_data,min_update_df,by = join_field) + records_to_insert <- meta_data records_inserted <- air_insert_data_frame(base, table_name,records_to_insert) From 339cd0e0051954cf1c6c945d3fcd12361db620fa Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 25 Apr 2023 16:00:40 -0600 Subject: [PATCH 106/126] using schema for col check in case table is empty --- R/air_dump.R | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/R/air_dump.R b/R/air_dump.R index 1cfe4cd..bd043c6 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -292,9 +292,11 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j schema <- air_get_schema(base) # check for Meta Data table + check_for_md_table <- schema$tables$name %in% table_name ## if no meta data table, stop - if(!table_name %in% schema$tables$name){ + + if(!all(check_for_md_table)){ msg <- glue::glue("No table called {table_name} in base {base}. Please use air_create_metadata_table to create the metadata table or create it manually.") @@ -319,20 +321,12 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j records_deleted = NA ) - # if the current metadata table is empty, then insert records - if(is.character(current_metadata_table)){ - message("added new records") - records_to_insert <- meta_data - - records_inserted <- air_insert_data_frame(base, table_name,records_to_insert) - - update_log$records_inserted <- records_inserted - - return(update_log) - } message("checking if any fields need to be added") - col_check <- !names(meta_data) %in% names(current_metadata_table) + + # use schema in case table is empty + current_col_names <- schema$tables$fields[md_check][[1]]$name + col_check <- !names(meta_data) %in% current_col_names if(all(col_check)){ msg <- glue::glue("The meta_data object and the metadata table in your base, {table_name}, share @@ -354,6 +348,18 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j } + # if the current metadata table is empty, then insert records + if(is.character(current_metadata_table)){ + message("added new records") + records_to_insert <- meta_data + + records_inserted <- air_insert_data_frame(base, table_name,records_to_insert) + + update_log$records_inserted <- records_inserted + + return(update_log) + } + # compare with updated values ## use field ids message("checking which fields need to be updated") From 981a43527b2e597807660b12f298a20f640ceed7 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 25 Apr 2023 16:03:27 -0600 Subject: [PATCH 107/126] updated md table check --- R/air_dump.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/air_dump.R b/R/air_dump.R index bd043c6..530539c 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -296,7 +296,7 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j ## if no meta data table, stop - if(!all(check_for_md_table)){ + if(!any(check_for_md_table)){ msg <- glue::glue("No table called {table_name} in base {base}. Please use air_create_metadata_table to create the metadata table or create it manually.") From ac5115bdb95f59f233e5b65c56ca705a2af2cfab Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 25 Apr 2023 16:05:01 -0600 Subject: [PATCH 108/126] fixed typo --- R/air_dump.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/air_dump.R b/R/air_dump.R index 530539c..4ef135a 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -325,7 +325,7 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j message("checking if any fields need to be added") # use schema in case table is empty - current_col_names <- schema$tables$fields[md_check][[1]]$name + current_col_names <- schema$tables$fields[check_for_md_table][[1]]$name col_check <- !names(meta_data) %in% current_col_names if(all(col_check)){ From 0aeeac29f40a7ec568d71e03752f6a826185312b Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 23 May 2023 16:03:18 -0600 Subject: [PATCH 109/126] added remotes section to description for deposits --- DESCRIPTION | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index a17b735..77446eb 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.9 +Version: 0.2.10 Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes @@ -27,6 +27,8 @@ Imports: stringr, tibble, deposits +Remotes: + ropenscilabs/deposits RoxygenNote: 7.2.3 Suggests: knitr, From 662539715964ddfbdef8350f835d9897b634b8ae Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 12 Jun 2023 15:22:31 -0600 Subject: [PATCH 110/126] added manual id feature --- DESCRIPTION | 2 +- R/air_dump.R | 9 +++++++-- man/air_dump_to_csv.Rd | 6 +++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 77446eb..0020fb9 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.10 +Version: 0.2.11 Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes diff --git a/R/air_dump.R b/R/air_dump.R index 4ef135a..772009d 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -1176,13 +1176,18 @@ flatten_col_to_chr <- function(data_frame){ #' @param output_dir String. Folder containing output files #' @param overwrite Logical. Should outputs be overwritten if they already exist? #' @param attachments_dir String. What folder are base attachments stored in? +#' @param output_id String. Optional identifier for the data set - if NULL an +#' ID will be generated using a hash of the data. #' #' @return Vector of file paths #' @export -air_dump_to_csv <- function(table_list,output_dir= "outputs",attachments_dir=NULL, overwrite = FALSE){ +air_dump_to_csv <- function(table_list,output_dir= "outputs",attachments_dir=NULL, overwrite = FALSE, output_id = NULL){ # create a unique id for the data - output_id <- rlang::hash(table_list) + if(is.null(output_id)){ + output_id <- rlang::hash(table_list) + } + # check if data already exist output_dir_path_final <- sprintf("%s/%s",output_dir,output_id) diff --git a/man/air_dump_to_csv.Rd b/man/air_dump_to_csv.Rd index 4a916ae..92c1385 100644 --- a/man/air_dump_to_csv.Rd +++ b/man/air_dump_to_csv.Rd @@ -8,7 +8,8 @@ air_dump_to_csv( table_list, output_dir = "outputs", attachments_dir = NULL, - overwrite = FALSE + overwrite = FALSE, + output_id = NULL ) } \arguments{ @@ -19,6 +20,9 @@ air_dump_to_csv( \item{attachments_dir}{String. What folder are base attachments stored in?} \item{overwrite}{Logical. Should outputs be overwritten if they already exist?} + +\item{output_id}{String. Optional identifier for the data set - if NULL an +ID will be generated using a hash of the data.} } \value{ Vector of file paths From 69dd10ab668497d204eafd1283da45d3ef71fe6a Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Wed, 14 Jun 2023 14:44:06 -0600 Subject: [PATCH 111/126] updated dump to csv to allow for non-snake case outputs --- DESCRIPTION | 2 +- R/air_dump.R | 72 +++++++++++++++++++++++++++--------------- man/air_dump_to_csv.Rd | 5 ++- 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 0020fb9..560d137 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.11 +Version: 0.2.12 Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes diff --git a/R/air_dump.R b/R/air_dump.R index 772009d..c54924f 100644 --- a/R/air_dump.R +++ b/R/air_dump.R @@ -392,9 +392,9 @@ air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", j if(nrow(records_to_delete) >0){ message("Deleting records no longer in the base") - records_deleted <- purrr::map(records_to_delete$id, function(id){ - air_delete(base, table_name,id) - }) + records_deleted <- purrr::map(records_to_delete$id, function(id){ + air_delete(base, table_name,id) + }) } else { message("No Records deleted") records_deleted <- "No records deleted" @@ -497,7 +497,7 @@ air_update_description_table <- function(base,description, table_name = "Descrip # convert to tibble for more consistent behavior in joins current_metadata_table<- tibble::as_tibble(current_metadata_table) current_metadata_table <- current_metadata_table |> - dplyr::select(-createdTime) + dplyr::select(-createdTime) # compare with updated values ## use field ids @@ -725,7 +725,7 @@ air_generate_metadata_from_api <- function(base, "Field ID", "Table ID", "Field Opts", "Primary Key") } - return(md_df) + return(md_df) }) return(metadata_df) @@ -795,7 +795,7 @@ air_get_base_description_from_table<- function(base, table_name,field_names_to_s desc_table <- airtabler::fetch_all(base,table_name) # to snake case if(field_names_to_snakecase){ - names(desc_table) <- snakecase::to_snake_case(names(desc_table)) + names(desc_table) <- snakecase::to_snake_case(names(desc_table)) } required_fields <- c("title","primary_contact","email","description") @@ -865,15 +865,15 @@ air_generate_base_description <- function(title = NA, identifier =NA, license = NA,...){ desc_table <- tibble::tibble(title = title, - creator= creator, - created=created, - primary_contact=primary_contact, - email = email, - description = description, - contributor = contributor, - identifier =identifier, - license = license, - ...) + creator= creator, + created=created, + primary_contact=primary_contact, + email = email, + description = description, + contributor = contributor, + identifier =identifier, + license = license, + ...) return(desc_table) } @@ -1035,7 +1035,7 @@ air_dump <- function(base, metadata= NULL, description = NULL, ## build up attachment fields on x_table sleep_time <- 0 if(polite_downloads){ - sleep_time <- 0.01 + sleep_time <- 0.01 } for(af in attachment_fields){ Sys.sleep(sleep_time) @@ -1075,9 +1075,9 @@ air_dump <- function(base, metadata= NULL, description = NULL, } } -# named_description_post <- grepl(pattern = "description",x = names(table_list), ignore.case = TRUE) -# -# table_list[named_description_post][[1]]$created <- Sys.Date() + # named_description_post <- grepl(pattern = "description",x = names(table_list), ignore.case = TRUE) + # + # table_list[named_description_post][[1]]$created <- Sys.Date() return(table_list) } @@ -1178,10 +1178,15 @@ flatten_col_to_chr <- function(data_frame){ #' @param attachments_dir String. What folder are base attachments stored in? #' @param output_id String. Optional identifier for the data set - if NULL an #' ID will be generated using a hash of the data. +#' @param names_to_snake_case Logical. Should field and table names be converted to snake_case? #' #' @return Vector of file paths #' @export -air_dump_to_csv <- function(table_list,output_dir= "outputs",attachments_dir=NULL, overwrite = FALSE, output_id = NULL){ +air_dump_to_csv <- function(table_list,output_dir= "outputs", + attachments_dir=NULL, + overwrite = FALSE, + output_id = NULL, + names_to_snake_case = TRUE){ # create a unique id for the data if(is.null(output_id)){ @@ -1206,14 +1211,29 @@ air_dump_to_csv <- function(table_list,output_dir= "outputs",attachments_dir=NUL dir.create(output_dir_path,recursive = TRUE) purrr::walk2(table_list, names(table_list), function(x_table,y_table_name){ - ## clean table name - y_table_name <- snakecase::to_snake_case(y_table_name) - ## clean up field names in table - names(x_table) <- snakecase::to_snake_case(names(x_table)) - ## clean up field names - names(x_table) <- snakecase::to_snake_case(names(x_table)) + if(names_to_snake_case){ + ## clean table name + y_table_name_snake <- snakecase::to_snake_case(y_table_name) + ## clean up field names in table + x_names_snake <- snakecase::to_snake_case(names(x_table)) + + ## check that we didn't create duplicate field names + dup_check <- duplicated(names(x_table)) + + if(any(dup_check)){ + err_msg <- glue::glue("The following field names in table {y_table_name} are duplicated after converting to snake_case: + \n + {names(x_table)[dup_check]}\n + Fix field names in airtable or set `names_to_snake_case` to FALSE") + rlang::abort(err_msg) + } + + y_table_name <- y_table_name_snake + names(x_table) <- x_names_snake + + } x_table_flat <- flatten_col_to_chr(x_table) ## export to CSV diff --git a/man/air_dump_to_csv.Rd b/man/air_dump_to_csv.Rd index 92c1385..4120fbd 100644 --- a/man/air_dump_to_csv.Rd +++ b/man/air_dump_to_csv.Rd @@ -9,7 +9,8 @@ air_dump_to_csv( output_dir = "outputs", attachments_dir = NULL, overwrite = FALSE, - output_id = NULL + output_id = NULL, + names_to_snake_case = TRUE ) } \arguments{ @@ -23,6 +24,8 @@ air_dump_to_csv( \item{output_id}{String. Optional identifier for the data set - if NULL an ID will be generated using a hash of the data.} + +\item{names_to_snake_case}{Logical. Should field and table names be converted to snake_case?} } \value{ Vector of file paths From 779dbbf9707121d64a4f14d3fa53b30a768d0b15 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 29 Jun 2023 08:04:45 -0500 Subject: [PATCH 112/126] added option for dropping attachement names --- R/air_download_attachments.R | 18 ++++++++++++++---- man/air_download_attachments.Rd | 11 ++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/R/air_download_attachments.R b/R/air_download_attachments.R index f7ec90f..9a7757d 100644 --- a/R/air_download_attachments.R +++ b/R/air_download_attachments.R @@ -10,6 +10,8 @@ #' @param dir_name String. Where should files be downloaded to? #' Will create the folder if it does not exist. Folders created are recursively. #' @param ... reserved for additional arguments. +#' @param include_attachment_id Logical. Should you include the airtable attachment +#' ID to guarantee all file names are unique? Default is true. #' #' @return Returns x with an additional field called attachment_file_paths #' @export air_download_attachments @@ -30,7 +32,7 @@ #' #' } #' -air_download_attachments <- function(x, field, dir_name = "downloads",...){ +air_download_attachments <- function(x, field, dir_name = "downloads",include_attachment_id = TRUE,...){ #browser() if(!is.data.frame(x)){ @@ -80,7 +82,16 @@ air_download_attachments <- function(x, field, dir_name = "downloads",...){ # with different contents - e.g. original file generation was # structured like sample_1234/fasta.file sample_1235/fasta.file - dest <- sprintf("%s/%s_%s", dir_name,x$id,x$filename) + filename_dest <- x$filename + + if(include_attachment_id){ + filename_dest <- sprintf("%s_%s", x$id,filename_dest) + } else { + message("include_attachement_id = FALSE. If file names are repeated, + only the first file with that name will be downloaded.") + } + + dest <- sprintf("%s/%s", dir_name,filename_dest) # sometimes the same file is attached multiple times # if the file is already downloaded, don't add it again @@ -89,8 +100,7 @@ air_download_attachments <- function(x, field, dir_name = "downloads",...){ if(all(file.exists(dest))){ - - not_downloaded_message <- glue::glue("\nFile already exists, not downloaded\n{dest}\n") + not_downloaded_message <- glue::glue("\nFile already exists, not downloaded\n{dest}\n.") print(not_downloaded_message) return(dest) } diff --git a/man/air_download_attachments.Rd b/man/air_download_attachments.Rd index 7b467cc..793bdbd 100644 --- a/man/air_download_attachments.Rd +++ b/man/air_download_attachments.Rd @@ -4,7 +4,13 @@ \alias{air_download_attachments} \title{Download Airtable file attachments} \usage{ -air_download_attachments(x, field, dir_name = "downloads", ...) +air_download_attachments( + x, + field, + dir_name = "downloads", + include_attachment_id = TRUE, + ... +) } \arguments{ \item{x}{Data frame. Output from air_get or fetch_all.} @@ -14,6 +20,9 @@ air_download_attachments(x, field, dir_name = "downloads", ...) \item{dir_name}{String. Where should files be downloaded to? Will create the folder if it does not exist. Folders created are recursively.} +\item{include_attachment_id}{Logical. Should you include the airtable attachment +ID to guarantee all file names are unique? Default is true.} + \item{...}{reserved for additional arguments.} } \value{ From 696d5084eac50f38086ed8beb5038eaebe87af1a Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 29 Jun 2023 08:05:46 -0500 Subject: [PATCH 113/126] updated version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 560d137..57ec73c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.12 +Version: 0.2.13 Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes From c08fd282a922fc352f79216acc07c9c517761782 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 18 Jul 2023 14:37:10 -0600 Subject: [PATCH 114/126] updated air_get_attachements --- DESCRIPTION | 2 +- R/air_get_attachments.R | 6 ++++-- man/air_get_attachments.Rd | 4 ++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 57ec73c..9ed3f0f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.13 +Version: 0.2.14 Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes diff --git a/R/air_get_attachments.R b/R/air_get_attachments.R index 0b40237..705fd8f 100644 --- a/R/air_get_attachments.R +++ b/R/air_get_attachments.R @@ -14,6 +14,8 @@ #' @param extract_field String. Name of extract field that will be created #' @param ... Additional arguments to pass to \code{air_get} #' @param download_file Logical. Should files be downloaded? +#' @param include_attachment_id Logical. Should the attachment ID be included in the file name? +#' Default is true to ensure unique file names. #' @param dir_name String. Where should files be downloaded to? #' Will create the folder if it does not exist. #' @param skip Numeric. How many lines should be skipped? See \code{readxl::read_excel} skip. @@ -34,7 +36,7 @@ #' } #' #' -air_get_attachments <- function(base, table_name, field, download_file = FALSE, dir_name = "downloads", extract_type ="excel", extract_field ="excel_extract", skip = 0, parse_all_sheets = FALSE, ...){ +air_get_attachments <- function(base, table_name, field, download_file = FALSE,include_attachment_id = TRUE, dir_name = "downloads", extract_type ="excel", extract_field ="excel_extract", skip = 0, parse_all_sheets = FALSE, ...){ # get data x <- fetch_all(base,table_name,...) @@ -47,7 +49,7 @@ air_get_attachments <- function(base, table_name, field, download_file = FALSE, ### get files ---- if(download_file){ - x <- air_download_attachments(x,field = field,dir_name = dir_name) + x <- air_download_attachments(x,field = field,dir_name = dir_name,include_attachment_id ) } ### extract excel ---- diff --git a/man/air_get_attachments.Rd b/man/air_get_attachments.Rd index ee6f104..efad70f 100644 --- a/man/air_get_attachments.Rd +++ b/man/air_get_attachments.Rd @@ -9,6 +9,7 @@ air_get_attachments( table_name, field, download_file = FALSE, + include_attachment_id = TRUE, dir_name = "downloads", extract_type = "excel", extract_field = "excel_extract", @@ -26,6 +27,9 @@ air_get_attachments( \item{download_file}{Logical. Should files be downloaded?} +\item{include_attachment_id}{Logical. Should the attachment ID be included in the file name? +Default is true to ensure unique file names.} + \item{dir_name}{String. Where should files be downloaded to? Will create the folder if it does not exist.} From 72ed9b802bff9f555faa50dca2eb23fa74ece7c9 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 21 Jul 2023 12:57:37 -0600 Subject: [PATCH 115/126] added update field function --- DESCRIPTION | 7 ++- R/air_metadata_api.R | 127 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 3 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 9ed3f0f..96e4500 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.14 +Version: 0.2.15 Date: 2022-12-15 Author: Darko Bergant Maintainer: Collin Schwantes @@ -26,8 +26,9 @@ Imports: rlang, stringr, tibble, - deposits -Remotes: + deposits, + assertthat +Remotes: ropenscilabs/deposits RoxygenNote: 7.2.3 Suggests: diff --git a/R/air_metadata_api.R b/R/air_metadata_api.R index 795310a..0919901 100644 --- a/R/air_metadata_api.R +++ b/R/air_metadata_api.R @@ -451,6 +451,132 @@ air_create_field <- function(base, } + +#' Update a field name and/or description +#' +#' Must update either the name or the description. +#' See "https://airtable.com/developers/web/api/update-field" for more details. +#' +#' @param base String. Base id +#' @param table_id String. ID for table that contains the field to be updated +#' @param field_id String. ID of field to be updated +#' @param name String. updated name (optional) +#' @param description String. updated description (option) +#' +#' @return List. Describes the changes that happened to the field +#' @export air_update_field +#' +#' @examples +#' \dontrun{ +#' base <- "appVjIfAo8AJlfTkx" +#' +#' schema <- air_get_schema("appVjIfAo8AJlfTkx") +#' +#' table_id <- schema$tables[1,c("id")] +#' +#' field_id <- schema$tables$fields[[1]][2,]$id +#' +#' ## update name and description +#' +#' name <- "New Name" +#' +#' description <- "Updated Description" +#' +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id,name = name, description = description) +#' +#' ### just name +#' +#' name <- "New New Name" +#' +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id,name = name) +#' +#' +#' ## just description +#' +#' description <- "Better description" +#' +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id,description = description) +#' +#' ## set name to number +#' +#' name <- 1234 +#' +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id,name = name) +#' +#' +#' # set description to number +#' +#' description <- 1234 +#' +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id,description = description) +#' +#' # bulk update names and descriptions from a data frame +#' +#' field_ids <- schema$tables$fields[[1]]$id +#' +#' field_names <- sprintf("%s_bulk_update",schema$tables$fields[[1]]$name) +#' +#' field_descriptions <- sprintf("%s BULK UPDATE",schema$tables$fields[[1]]$description) +#' +#' df <- data.frame("field_id"= field_ids,"name"=field_names,"description"=field_descriptions) +#' +#' purrr::pmap(df,function(field_id,name,description){ +#' air_update_field(base = base,table_id = table_id,field_id = field_id,name = name, description = description) +#' }) +#' } +air_update_field<- function(base, table_id, field_id, name = NULL, description = NULL){ + + if(rlang::is_empty(name) & rlang::is_empty(description)){ + rlang::abort("Must provide either a name or description") + } + + ## create the field list + field_list<- list() + + # add name property + if(!rlang::is_empty(name)){ + assertthat::assert_that(is.character(name),length(name) ==1) + field_list$name <- name + } + + # add description property + if(!rlang::is_empty(description)){ + assertthat::assert_that(is.character(description),length(description) ==1) + field_list$description <- description + } + + assertthat::assert_that(length(field_list) > 0) + + # create + "https://api.airtable.com/v0/meta/bases/{baseId}/tables/{tableId}/fields/{columnId}" + + request_url <- sprintf("%s/%s/tables/%s/fields/%s", air_meta_url, base,table_id,field_id) + request_url <- utils::URLencode(request_url) + + + ## fields must be updated one at a time + fields_json <- jsonlite::toJSON(field_list,pretty = TRUE,auto_unbox = TRUE) + + # call service: + res <- httr::PATCH( + request_url, + httr::content_type("application/json"), + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ), + body = fields_json + ) + + air_validate(res) + # may need a new air_parse function + + res_content <- httr::content(res,as = "text") + + schema <- jsonlite::fromJSON(res_content) + + return(schema) +} + #' Get list of bases for an Token #' #' Each token you provisision is given access to a certain set of bases or @@ -491,3 +617,4 @@ air_list_bases <- function(request_url = "https://api.airtable.com/v0/meta/bases } + From 538f1eac58d24560aae9eb59f869d3c293ae218b Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Fri, 21 Jul 2023 14:12:30 -0600 Subject: [PATCH 116/126] added documentation files --- NAMESPACE | 1 + man/air_update_field.Rd | 85 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 man/air_update_field.Rd diff --git a/NAMESPACE b/NAMESPACE index 1817868..e96d160 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -32,6 +32,7 @@ export(air_table_template) export(air_update) export(air_update_data_frame) export(air_update_description_table) +export(air_update_field) export(air_update_metadata_table) export(airtable) export(fetch_all) diff --git a/man/air_update_field.Rd b/man/air_update_field.Rd new file mode 100644 index 0000000..333a2e8 --- /dev/null +++ b/man/air_update_field.Rd @@ -0,0 +1,85 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_update_field} +\alias{air_update_field} +\title{Update a field name and/or description} +\usage{ +air_update_field(base, table_id, field_id, name = NULL, description = NULL) +} +\arguments{ +\item{base}{String. Base id} + +\item{table_id}{String. ID for table that contains the field to be updated} + +\item{field_id}{String. ID of field to be updated} + +\item{name}{String. updated name (optional)} + +\item{description}{String. updated description (option)} +} +\value{ +List. Describes the changes that happened to the field +} +\description{ +Must update either the name or the description. +See "https://airtable.com/developers/web/api/update-field" for more details. +} +\examples{ +\dontrun{ +base <- "appVjIfAo8AJlfTkx" + +schema <- air_get_schema("appVjIfAo8AJlfTkx") + +table_id <- schema$tables[1,c("id")] + +field_id <- schema$tables$fields[[1]][2,]$id + +## update name and description + +name <- "New Name" + +description <- "Updated Description" + +out <- air_update_field(base = base,table_id = table_id,field_id = field_id,name = name, description = description) + +### just name + +name <- "New New Name" + +out <- air_update_field(base = base,table_id = table_id,field_id = field_id,name = name) + + +## just description + +description <- "Better description" + +out <- air_update_field(base = base,table_id = table_id,field_id = field_id,description = description) + +## set name to number + +name <- 1234 + +out <- air_update_field(base = base,table_id = table_id,field_id = field_id,name = name) + + +# set description to number + +description <- 1234 + +out <- air_update_field(base = base,table_id = table_id,field_id = field_id,description = description) + +# bulk update names and descriptions from a data frame + +field_ids <- schema$tables$fields[[1]]$id + +field_names <- sprintf("\%s_bulk_update",schema$tables$fields[[1]]$name) + +field_descriptions <- sprintf("\%s BULK UPDATE",schema$tables$fields[[1]]$description) + +df <- data.frame("field_id"= field_ids,"name"=field_names,"description"=field_descriptions) + +purrr::pmap(df,function(field_id,name,description){ + air_update_field(base = base,table_id = table_id,field_id = field_id,name = name, description = description) +}) +} +} From 051b7784ff202bb25b9d6bb0e56f5c764e947cb3 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Wed, 2 Aug 2023 16:40:14 -0600 Subject: [PATCH 117/126] updated documentation for air_get --- R/airtabler.R | 4 ++-- man/air_get.Rd | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/airtabler.R b/R/airtabler.R index ad22ce8..6eef229 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -82,8 +82,8 @@ air_secret_key <- function(){ #' records will be returned. Defaults to asc. #' @param combined_result If TRUE (default) all data is returned in the same data. #' If FALSE table fields are returned in separate \code{fields} element. -#' @param fields (optional) Only data for fields whose names are in this list -#' will be included in the records. If you don't need every field, you can use +#' @param fields List. (optional) Only data for fields whose names are in this list +#' will be included in the records. Does not work when retrieving individual records with \code{record_id} #' @param filterByFormula String. Use a formula to filter results. See \href{airtable docs}{https://support.airtable.com/hc/en-us/articles/223247187-How-to-sort-filter-or-retrieve-ordered-records-in-the-API} #' this parameter to reduce the amount of data transferred. #' @return A data frame with records or a list with record details if diff --git a/man/air_get.Rd b/man/air_get.Rd index 37a501a..97bfe47 100644 --- a/man/air_get.Rd +++ b/man/air_get.Rd @@ -34,8 +34,8 @@ call. Note that this is represented by a record ID, not a numerical offset.} \item{view}{(optional) The name or ID of the view} -\item{fields}{(optional) Only data for fields whose names are in this list -will be included in the records. If you don't need every field, you can use} +\item{fields}{List. (optional) Only data for fields whose names are in this list +will be included in the records. Does not work when retrieving individual records with \code{record_id}} \item{sortField}{(optional) The field name to use for sorting} From 140911047ca650c0bc10fc3c2efd1266e895b07f Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 3 Aug 2023 15:50:20 -0600 Subject: [PATCH 118/126] updating documentation --- NAMESPACE | 5 +++++ R/air_metadata_api.R | 24 ++++++++++++++++-------- R/fetch_all.R | 8 ++++++-- man/air_update_field.Rd | 24 ++++++++++++++++-------- man/fetch_all.Rd | 8 ++++++-- 5 files changed, 49 insertions(+), 20 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index e96d160..dbe6e7e 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -19,9 +19,14 @@ export(air_generate_metadata_from_tables) export(air_get) export(air_get_attachments) export(air_get_base_description_from_table) +export(air_get_base_id_from_url) +export(air_get_id_from_url) export(air_get_json) export(air_get_metadata_from_table) +export(air_get_record_id_from_url) export(air_get_schema) +export(air_get_table_id_from_url) +export(air_get_view_id_from_url) export(air_insert) export(air_insert_data_frame) export(air_list_bases) diff --git a/R/air_metadata_api.R b/R/air_metadata_api.R index 0919901..a5d0e6e 100644 --- a/R/air_metadata_api.R +++ b/R/air_metadata_api.R @@ -482,33 +482,38 @@ air_create_field <- function(base, #' #' description <- "Updated Description" #' -#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id,name = name, description = description) +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +#' name = name, description = description) #' #' ### just name #' #' name <- "New New Name" #' -#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id,name = name) +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +#' name = name) #' #' #' ## just description #' #' description <- "Better description" #' -#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id,description = description) +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +#' description = description) #' #' ## set name to number #' #' name <- 1234 #' -#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id,name = name) +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +#' name = name) #' #' #' # set description to number #' #' description <- 1234 #' -#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id,description = description) +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +#' description = description) #' #' # bulk update names and descriptions from a data frame #' @@ -516,12 +521,15 @@ air_create_field <- function(base, #' #' field_names <- sprintf("%s_bulk_update",schema$tables$fields[[1]]$name) #' -#' field_descriptions <- sprintf("%s BULK UPDATE",schema$tables$fields[[1]]$description) +#' field_descriptions <- sprintf("%s BULK UPDATE", +#' schema$tables$fields[[1]]$description) #' -#' df <- data.frame("field_id"= field_ids,"name"=field_names,"description"=field_descriptions) +#' df <- data.frame("field_id"= field_ids,"name"=field_names, +#' "description"=field_descriptions) #' #' purrr::pmap(df,function(field_id,name,description){ -#' air_update_field(base = base,table_id = table_id,field_id = field_id,name = name, description = description) +#' air_update_field(base = base,table_id = table_id,field_id = field_id, +#' name = name, description = description) #' }) #' } air_update_field<- function(base, table_id, field_id, name = NULL, description = NULL){ diff --git a/R/fetch_all.R b/R/fetch_all.R index cc60363..1f5ff88 100644 --- a/R/fetch_all.R +++ b/R/fetch_all.R @@ -17,12 +17,16 @@ #' #' \code{AIRTABLE_API_KEY=your_api_key_here} #' -#' You can use \code{usethis::edit_r_environ()} to edit your find and edit your +#' You can use \code{usethis::edit_r_environ()} to open and edit your .Renviron #' file. #' +#' Also consider using the `dotenv` package with a .env file for storing +#' sensitive variables. Remember add to.gitignore or encrypt the .env file to +#' avoid sharing sensitive variables. +#' #' @param base String. ID for the base or app to be fetched #' @param table_name String. Name of the table to be fetched from the base -#' @param ... Additional arguments to pass to \code{air_get}. \code{view} is a +#' @param ... Additional arguments to pass to [air_get()]. \code{view} is a #' commonly used additional argument. #' #' @return dataframe diff --git a/man/air_update_field.Rd b/man/air_update_field.Rd index 333a2e8..4caafec 100644 --- a/man/air_update_field.Rd +++ b/man/air_update_field.Rd @@ -40,33 +40,38 @@ name <- "New Name" description <- "Updated Description" -out <- air_update_field(base = base,table_id = table_id,field_id = field_id,name = name, description = description) +out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +name = name, description = description) ### just name name <- "New New Name" -out <- air_update_field(base = base,table_id = table_id,field_id = field_id,name = name) +out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +name = name) ## just description description <- "Better description" -out <- air_update_field(base = base,table_id = table_id,field_id = field_id,description = description) +out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +description = description) ## set name to number name <- 1234 -out <- air_update_field(base = base,table_id = table_id,field_id = field_id,name = name) +out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +name = name) # set description to number description <- 1234 -out <- air_update_field(base = base,table_id = table_id,field_id = field_id,description = description) +out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +description = description) # bulk update names and descriptions from a data frame @@ -74,12 +79,15 @@ field_ids <- schema$tables$fields[[1]]$id field_names <- sprintf("\%s_bulk_update",schema$tables$fields[[1]]$name) -field_descriptions <- sprintf("\%s BULK UPDATE",schema$tables$fields[[1]]$description) +field_descriptions <- sprintf("\%s BULK UPDATE", +schema$tables$fields[[1]]$description) -df <- data.frame("field_id"= field_ids,"name"=field_names,"description"=field_descriptions) +df <- data.frame("field_id"= field_ids,"name"=field_names, +"description"=field_descriptions) purrr::pmap(df,function(field_id,name,description){ - air_update_field(base = base,table_id = table_id,field_id = field_id,name = name, description = description) + air_update_field(base = base,table_id = table_id,field_id = field_id, + name = name, description = description) }) } } diff --git a/man/fetch_all.Rd b/man/fetch_all.Rd index 3f9e4c5..62e0d1f 100644 --- a/man/fetch_all.Rd +++ b/man/fetch_all.Rd @@ -11,7 +11,7 @@ fetch_all(base, table_name, ...) \item{table_name}{String. Name of the table to be fetched from the base} -\item{...}{Additional arguments to pass to \code{air_get}. \code{view} is a +\item{...}{Additional arguments to pass to [air_get()]. \code{view} is a commonly used additional argument.} } \value{ @@ -36,8 +36,12 @@ home directory with a line like this: \code{AIRTABLE_API_KEY=your_api_key_here} -You can use \code{usethis::edit_r_environ()} to edit your find and edit your +You can use \code{usethis::edit_r_environ()} to open and edit your .Renviron file. + +Also consider using the `dotenv` package with a .env file for storing +sensitive variables. Remember add to.gitignore or encrypt the .env file to +avoid sharing sensitive variables. } \examples{ From 2367267004f002385638a6cd48d343d7f6638da8 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Tue, 15 Aug 2023 09:29:22 -0600 Subject: [PATCH 119/126] adding files for get_id functions --- R/air_metadata_api.R | 2 +- R/get_id_from_url.R | 77 ++++++++++++++++++++++++++++++++++++++ man/air_get_id_from_url.Rd | 59 +++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 R/get_id_from_url.R create mode 100644 man/air_get_id_from_url.Rd diff --git a/R/air_metadata_api.R b/R/air_metadata_api.R index a5d0e6e..8c703ed 100644 --- a/R/air_metadata_api.R +++ b/R/air_metadata_api.R @@ -342,7 +342,7 @@ air_table_template <- function(table_name, description, fields_df ){ #' #' table_list <- air_table_template(table_name = "Planet", #' description = "Planets of Foundation", -#' fields_df = field_tables) +#' fields_df = field_df) #' #' air_create_table(base, table_list) #' diff --git a/R/get_id_from_url.R b/R/get_id_from_url.R new file mode 100644 index 0000000..db953fa --- /dev/null +++ b/R/get_id_from_url.R @@ -0,0 +1,77 @@ +#' Get an ID from a URL +#' +#' General function for parsing airtable URLs to find base, table, view, or record id's +#' +#' @param url String. A url generated by airtable +#' @param pattern String. A regex pattern for identifying the type of id you would like to get +#' @param id_type String. One of base_id, table_id, view_id, or record_id. +#' @param split_pattern String. Where should the URL be split? default is forward slashes "/" and questionmarks "?" +#' +#' @return String +#' @export air_get_id_from_url +#' +#' @examples +#' +#' url <- "https://airtable.com/apphDEokVZ9gvPNFk/tblaKC1ADBafoHVXN/viwteUgD7vaMBruHR/recMzdoM43RVRWybD?blocks=hide" +#' +#' # General function for parsing url components +#' air_get_id_from_url(url, '^app',id_type = "base_id") +#' # Get different components +#' air_get_base_id_from_url(url) +#' air_get_table_id_from_url(url) +#' air_get_view_id_from_url(url) +#' air_get_record_id_from_url(url) +#' +air_get_id_from_url <- function(url, pattern, id_type, split_pattern = "/|\\?" ){ + + url_split <- stringr::str_split_1(string = url,pattern = split_pattern) + + id_filter <- stringr::str_detect(url_split,pattern = pattern) + + if(all(!id_filter)){ + + err_msg <- glue::glue("{id_type} should conform to the pattern {pattern}.\nNo {id_type} found in {url}.") + + rlang::abort(err_msg) + } + + id <- url_split[id_filter] + + return(id) +} + + + +#' @describeIn air_get_id_from_url Get the base id +#' @export +#' +air_get_base_id_from_url <- function(url,pattern = "^app\\w{13}"){ + + id <- air_get_id_from_url(url = url, pattern = pattern,id_type = "base_id") + + return(id) +} + +#' @describeIn air_get_id_from_url Get the table id +#' @export +#' +air_get_table_id_from_url<- function(url, pattern = "^tbl\\w{13}"){ + id <- air_get_id_from_url(url = url, pattern = pattern,id_type = "table_id") + return(id) +} + +#' @describeIn air_get_id_from_url Get the view id +#' @export +#' +air_get_view_id_from_url<- function(url, pattern = "^viw\\w{13}"){ + id <- air_get_id_from_url(url = url, pattern = pattern,id_type = "view_id") + return(id) +} + +#' @describeIn air_get_id_from_url Get the record id +#' @export +#' +air_get_record_id_from_url<- function(url, pattern = "^rec\\w{13}"){ + id <- air_get_id_from_url(url = url, pattern = pattern,id_type = "record_id") + return(id) +} diff --git a/man/air_get_id_from_url.Rd b/man/air_get_id_from_url.Rd new file mode 100644 index 0000000..4304581 --- /dev/null +++ b/man/air_get_id_from_url.Rd @@ -0,0 +1,59 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_id_from_url.R +\name{air_get_id_from_url} +\alias{air_get_id_from_url} +\alias{air_get_base_id_from_url} +\alias{air_get_table_id_from_url} +\alias{air_get_view_id_from_url} +\alias{air_get_record_id_from_url} +\title{Get an ID from a URL} +\usage{ +air_get_id_from_url(url, pattern, id_type, split_pattern = "/|\\\\?") + +air_get_base_id_from_url(url, pattern = "^app\\\\w{13}") + +air_get_table_id_from_url(url, pattern = "^tbl\\\\w{13}") + +air_get_view_id_from_url(url, pattern = "^viw\\\\w{13}") + +air_get_record_id_from_url(url, pattern = "^rec\\\\w{13}") +} +\arguments{ +\item{url}{String. A url generated by airtable} + +\item{pattern}{String. A regex pattern for identifying the type of id you would like to get} + +\item{id_type}{String. One of base_id, table_id, view_id, or record_id.} + +\item{split_pattern}{String. Where should the URL be split? default is forward slashes "/" and questionmarks "?"} +} +\value{ +String +} +\description{ +General function for parsing airtable URLs to find base, table, view, or record id's +} +\section{Functions}{ +\itemize{ +\item \code{air_get_base_id_from_url()}: Get the base id + +\item \code{air_get_table_id_from_url()}: Get the table id + +\item \code{air_get_view_id_from_url()}: Get the view id + +\item \code{air_get_record_id_from_url()}: Get the record id + +}} +\examples{ + +url <- "https://airtable.com/apphDEokVZ9gvPNFk/tblaKC1ADBafoHVXN/viwteUgD7vaMBruHR/recMzdoM43RVRWybD?blocks=hide" + +# General function for parsing url components +air_get_id_from_url(url, '^app',id_type = "base_id") +# Get different components +air_get_base_id_from_url(url) +air_get_table_id_from_url(url) +air_get_view_id_from_url(url) +air_get_record_id_from_url(url) + +} From 1b3a015962a2bda88a4de1393cfab6e088b57929 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 17 Aug 2023 15:10:02 -0600 Subject: [PATCH 120/126] renamed a variable --- man/air_create_table.Rd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/air_create_table.Rd b/man/air_create_table.Rd index 0a604e9..9d7801c 100644 --- a/man/air_create_table.Rd +++ b/man/air_create_table.Rd @@ -63,7 +63,7 @@ field_df<- air_fields_df_template(name = field_names, table_list <- air_table_template(table_name = "Planet", description = "Planets of Foundation", - fields_df = field_tables) + fields_df = field_df) air_create_table(base, table_list) From 532e9a26c96f975de9644f60243c8eb756c7d8c2 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 17 Aug 2023 15:54:33 -0600 Subject: [PATCH 121/126] adding preliminary post function --- R/air_post.R | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++ R/airtabler.R | 9 ++++++ 2 files changed, 93 insertions(+) create mode 100644 R/air_post.R diff --git a/R/air_post.R b/R/air_post.R new file mode 100644 index 0000000..8d1030b --- /dev/null +++ b/R/air_post.R @@ -0,0 +1,84 @@ +#' Get a list of records +#' +#' Retrieve records where the request url would be over 16k characters (e.g. listing +#' many fields) +#' +#'You can retrieve records in an order of a view by providing the name or ID of +#' the view in the view query parameter. The results will include only records +#' visible in the order they are displayed. +#' +#' @param base String. Airtable base +#' @param table_name String. Table name +#' @param limit Numeric. (optional) A limit on the number of records to be returned. +#' Limit can range between 1 and 100. +#' @param offset Numeric. (optional) Page offset returned by the previous list-records +#' call. Note that this is represented by a record ID, not a numerical offset. +#' @param view String. (optional) The name or ID of the view +#' @param sortField String. (optional) The field name to use for sorting +#' @param sortDirection String. (optional) "asc" or "desc". The sort order in which the +#' records will be returned. Defaults to asc. +#' @param combined_result Logical. If TRUE (default) all data is returned in the same data. +#' If FALSE table fields are returned in separate \code{fields} element. +#' @param fields List. (optional) Only data for fields whose names are in this list +#' will be included in the records. Does not work when retrieving individual records with \code{record_id} +#' @param filterByFormula String. Use a formula to filter results. See \href{airtable docs}{https://support.airtable.com/hc/en-us/articles/223247187-How-to-sort-filter-or-retrieve-ordered-records-in-the-API} +#' this parameter to reduce the amount of data transferred. +#' @return A data frame with records +#' @export + +air_post <- function(base, + table_name, + limit = NULL, + offset = NULL, + view = NULL, + fields = NULL, + sortField = NULL, + sortDirection = NULL, + filterByFormula = NULL, + combined_result = TRUE){ + + search_path <- table_name + + request_url <- sprintf("%s/%s/%s/listRecords", air_url, base, search_path) + request_url <- utils::URLencode(request_url) + + # append parameters to URL: + param_list <- as.list(environment())[c( + "limit", "offset", "view", "sortField", "sortDirection","filterByFormula")] + param_list <- param_list[!sapply(param_list, is.null)] + if(!is.null(fields)) { + param_list <- c(param_list, list_params(x = fields, par_name = "fields")) + } + + #request_url <- httr::modify_url(request_url, query = param_list) + #request_url <- gsub(pattern = "fields=",replacement = "fields%5B%5D=",x = request_url) + + "Note Airtable's API only accepts request with a URL shorter + than 16,000 characters. Encoded formulas may cause your + requests to exceed this limit. To fix this issue you can + instead make a POST request to + /v0/{baseId}/{tableIdOrName}/listRecords while + passing the parameters within the body of the + request instead of the query parameters." + + res <- httr::POST( + url = request_url, + config = httr::add_headers(Authorization = paste("Bearer", air_api_key())), + body = jsonlite::toJSON(param_list), + encode="json" + ) + + air_validate(res) # throws exception (stop) if error + ret <- air_parse(res) # returns R object + if(combined_result) { + # combine ID, Fields and CreatedTime in the same data frame: + ret <- + cbind( + id = ret$id, ret$fields, createdTime = ret$createdTime, + stringsAsFactors =FALSE + ) + } + + return(ret) + +} diff --git a/R/airtabler.R b/R/airtabler.R index 6eef229..9be6630 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -119,6 +119,15 @@ air_get <- function(base, table_name, request_url <- httr::modify_url(request_url, query = param_list) request_url <- gsub(pattern = "fields=",replacement = "fields%5B%5D=",x = request_url) + "Note Airtable's API only accepts request with a URL shorter + than 16,000 characters. Encoded formulas may cause your + requests to exceed this limit. To fix this issue you can + instead make a POST request to + /v0/{baseId}/{tableIdOrName}/listRecords while + passing the parameters within the body of the + request instead of the query parameters." + + #print(request_url) # call service: res <- httr::GET( From ac506f30195caacfd963d5a12478c14d41e2dff5 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 17 Aug 2023 16:13:00 -0600 Subject: [PATCH 122/126] adding air_post and exposing get_id functions --- DESCRIPTION | 4 ++-- NAMESPACE | 1 + R/airtabler.R | 7 ++++-- R/get_id_from_url.R | 6 ++--- man/air_post.Rd | 58 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 man/air_post.Rd diff --git a/DESCRIPTION b/DESCRIPTION index 96e4500..65bae38 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.2.15 -Date: 2022-12-15 +Version: 0.2.16 +Date: 2023-08-17 Author: Darko Bergant Maintainer: Collin Schwantes Description: Fork from Darko Bergant's package. Provides access to the Airtable (airtable.com) API. diff --git a/NAMESPACE b/NAMESPACE index dbe6e7e..16d86ad 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -32,6 +32,7 @@ export(air_insert_data_frame) export(air_list_bases) export(air_make_json) export(air_make_request) +export(air_post) export(air_select) export(air_table_template) export(air_update) diff --git a/R/airtabler.R b/R/airtabler.R index 9be6630..885335c 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -127,8 +127,10 @@ air_get <- function(base, table_name, passing the parameters within the body of the request instead of the query parameters." - - #print(request_url) + if(nchar(request_url) > 1600 | length(fields) > 21){ + print("using air_post") + air_post(base,table_name,limit,offset,view,fields,sortField,sortDirection,filterByFormula,combined_result) + } else { # call service: res <- httr::GET( url = request_url, @@ -145,6 +147,7 @@ air_get <- function(base, table_name, ) } ret + } } list_params <- function(x, par_name) { diff --git a/R/get_id_from_url.R b/R/get_id_from_url.R index db953fa..3e51b81 100644 --- a/R/get_id_from_url.R +++ b/R/get_id_from_url.R @@ -43,7 +43,7 @@ air_get_id_from_url <- function(url, pattern, id_type, split_pattern = "/|\\?" ) #' @describeIn air_get_id_from_url Get the base id -#' @export +#' @export air_get_base_id_from_url #' air_get_base_id_from_url <- function(url,pattern = "^app\\w{13}"){ @@ -53,7 +53,7 @@ air_get_base_id_from_url <- function(url,pattern = "^app\\w{13}"){ } #' @describeIn air_get_id_from_url Get the table id -#' @export +#' @export air_get_table_id_from_url #' air_get_table_id_from_url<- function(url, pattern = "^tbl\\w{13}"){ id <- air_get_id_from_url(url = url, pattern = pattern,id_type = "table_id") @@ -69,7 +69,7 @@ air_get_view_id_from_url<- function(url, pattern = "^viw\\w{13}"){ } #' @describeIn air_get_id_from_url Get the record id -#' @export +#' @export air_get_record_id_from_url #' air_get_record_id_from_url<- function(url, pattern = "^rec\\w{13}"){ id <- air_get_id_from_url(url = url, pattern = pattern,id_type = "record_id") diff --git a/man/air_post.Rd b/man/air_post.Rd new file mode 100644 index 0000000..4be5edf --- /dev/null +++ b/man/air_post.Rd @@ -0,0 +1,58 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_post.R +\name{air_post} +\alias{air_post} +\title{Get a list of records} +\usage{ +air_post( + base, + table_name, + limit = NULL, + offset = NULL, + view = NULL, + fields = NULL, + sortField = NULL, + sortDirection = NULL, + filterByFormula = NULL, + combined_result = TRUE +) +} +\arguments{ +\item{base}{String. Airtable base} + +\item{table_name}{String. Table name} + +\item{limit}{Numeric. (optional) A limit on the number of records to be returned. +Limit can range between 1 and 100.} + +\item{offset}{Numeric. (optional) Page offset returned by the previous list-records +call. Note that this is represented by a record ID, not a numerical offset.} + +\item{view}{String. (optional) The name or ID of the view} + +\item{fields}{List. (optional) Only data for fields whose names are in this list +will be included in the records. Does not work when retrieving individual records with \code{record_id}} + +\item{sortField}{String. (optional) The field name to use for sorting} + +\item{sortDirection}{String. (optional) "asc" or "desc". The sort order in which the +records will be returned. Defaults to asc.} + +\item{filterByFormula}{String. Use a formula to filter results. See \href{airtable docs}{https://support.airtable.com/hc/en-us/articles/223247187-How-to-sort-filter-or-retrieve-ordered-records-in-the-API} +this parameter to reduce the amount of data transferred.} + +\item{combined_result}{Logical. If TRUE (default) all data is returned in the same data. +If FALSE table fields are returned in separate \code{fields} element.} +} +\value{ +A data frame with records +} +\description{ +Retrieve records where the request url would be over 16k characters (e.g. listing +many fields) +} +\details{ +You can retrieve records in an order of a view by providing the name or ID of +the view in the view query parameter. The results will include only records +visible in the order they are displayed. +} From 96d535ed079a563ded96cddbc828b36c31c6ea9d Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 17 Aug 2023 16:16:35 -0600 Subject: [PATCH 123/126] updated documentation --- R/air_post.R | 8 +++++--- man/air_post.Rd | 7 +++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/R/air_post.R b/R/air_post.R index 8d1030b..ea78fd9 100644 --- a/R/air_post.R +++ b/R/air_post.R @@ -1,12 +1,14 @@ #' Get a list of records #' -#' Retrieve records where the request url would be over 16k characters (e.g. listing -#' many fields) +#' Retrieve records where the request url would be over 16k characters (e.g. +#' complicated formula) or has more than 21 fields listed in the request. #' -#'You can retrieve records in an order of a view by providing the name or ID of +#' You can retrieve records in an order of a view by providing the name or ID of #' the view in the view query parameter. The results will include only records #' visible in the order they are displayed. #' +#' @seealso [air_get()] +#' #' @param base String. Airtable base #' @param table_name String. Table name #' @param limit Numeric. (optional) A limit on the number of records to be returned. diff --git a/man/air_post.Rd b/man/air_post.Rd index 4be5edf..afa1116 100644 --- a/man/air_post.Rd +++ b/man/air_post.Rd @@ -48,11 +48,14 @@ If FALSE table fields are returned in separate \code{fields} element.} A data frame with records } \description{ -Retrieve records where the request url would be over 16k characters (e.g. listing -many fields) +Retrieve records where the request url would be over 16k characters (e.g. +complicated formula) or has more than 21 fields listed in the request. } \details{ You can retrieve records in an order of a view by providing the name or ID of the view in the view query parameter. The results will include only records visible in the order they are displayed. } +\seealso{ +[air_get()] +} From 468a1b1a85656f50f21a7c4436771cd031183ee8 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Thu, 17 Aug 2023 16:26:48 -0600 Subject: [PATCH 124/126] dropped print statement from air_get --- R/airtabler.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/airtabler.R b/R/airtabler.R index 885335c..c959908 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -128,7 +128,7 @@ air_get <- function(base, table_name, request instead of the query parameters." if(nchar(request_url) > 1600 | length(fields) > 21){ - print("using air_post") + # print("using air_post") air_post(base,table_name,limit,offset,view,fields,sortField,sortDirection,filterByFormula,combined_result) } else { # call service: From cff8f6c3103c6600ab3a8072a8bb0495aef5a452 Mon Sep 17 00:00:00 2001 From: Collin Schwantes Date: Mon, 21 Aug 2023 17:34:55 -0600 Subject: [PATCH 125/126] properly formatted httr request --- R/air_post.R | 20 +++++++--- R/airtabler.R | 107 +++++++++++++++++++++++++------------------------- 2 files changed, 69 insertions(+), 58 deletions(-) diff --git a/R/air_post.R b/R/air_post.R index ea78fd9..163f98b 100644 --- a/R/air_post.R +++ b/R/air_post.R @@ -49,11 +49,15 @@ air_post <- function(base, "limit", "offset", "view", "sortField", "sortDirection","filterByFormula")] param_list <- param_list[!sapply(param_list, is.null)] if(!is.null(fields)) { - param_list <- c(param_list, list_params(x = fields, par_name = "fields")) + ## allows us to use in fetch_all without having to specify a method and have + # properly formatted json + fields_unlisted <- unlist(fields) + param_list <- c(param_list, list_params(x = list(fields_unlisted), par_name = "fields")) } - #request_url <- httr::modify_url(request_url, query = param_list) - #request_url <- gsub(pattern = "fields=",replacement = "fields%5B%5D=",x = request_url) + # needed to properly format param name in json + names(param_list) <- gsub(pattern = "\\[\\d\\]",replacement = "", names(param_list) ) + "Note Airtable's API only accepts request with a URL shorter than 16,000 characters. Encoded formulas may cause your @@ -66,12 +70,14 @@ air_post <- function(base, res <- httr::POST( url = request_url, config = httr::add_headers(Authorization = paste("Bearer", air_api_key())), - body = jsonlite::toJSON(param_list), - encode="json" + body = jsonlite::toJSON(param_list,auto_unbox = TRUE), + encode="raw", + httr::content_type_json() ) air_validate(res) # throws exception (stop) if error ret <- air_parse(res) # returns R object + ret_offset <- get_offset(ret) if(combined_result) { # combine ID, Fields and CreatedTime in the same data frame: ret <- @@ -79,6 +85,10 @@ air_post <- function(base, id = ret$id, ret$fields, createdTime = ret$createdTime, stringsAsFactors =FALSE ) + + if(!is.null(ret_offset)) { + attr(ret, "offset") <- ret_offset + } } return(ret) diff --git a/R/airtabler.R b/R/airtabler.R index c959908..402f672 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -90,15 +90,15 @@ air_secret_key <- function(){ #' \code{record_id} is specified. #' @export air_get <- function(base, table_name, - record_id = NULL, - limit = NULL, - offset = NULL, - view = NULL, - fields = NULL, - sortField = NULL, - sortDirection = NULL, - filterByFormula = NULL, - combined_result = TRUE) { + record_id = NULL, + limit = NULL, + offset = NULL, + view = NULL, + fields = NULL, + sortField = NULL, + sortDirection = NULL, + filterByFormula = NULL, + combined_result = TRUE) { search_path <- table_name @@ -129,24 +129,25 @@ air_get <- function(base, table_name, if(nchar(request_url) > 1600 | length(fields) > 21){ # print("using air_post") - air_post(base,table_name,limit,offset,view,fields,sortField,sortDirection,filterByFormula,combined_result) + ret <- air_post(base,table_name,limit,offset,view,fields,sortField,sortDirection,filterByFormula,combined_result) + return(ret) } else { - # call service: - res <- httr::GET( - url = request_url, - config = httr::add_headers(Authorization = paste("Bearer", air_api_key())) - ) - air_validate(res) # throws exception (stop) if error - ret <- air_parse(res) # returns R object - if(combined_result && is.null(record_id)) { - # combine ID, Fields and CreatedTime in the same data frame: - ret <- - cbind( - id = ret$id, ret$fields, createdTime = ret$createdTime, - stringsAsFactors =FALSE - ) - } - ret + # call service: + res <- httr::GET( + url = request_url, + config = httr::add_headers(Authorization = paste("Bearer", air_api_key())) + ) + air_validate(res) # throws exception (stop) if error + ret <- air_parse(res) # returns R object + if(combined_result && is.null(record_id)) { + # combine ID, Fields and CreatedTime in the same data frame: + ret <- + cbind( + id = ret$id, ret$fields, createdTime = ret$createdTime, + stringsAsFactors =FALSE + ) + } + return(ret) } } @@ -220,15 +221,15 @@ list_params <- function(x, par_name) { #' \code{record_id} is specified. #' @export air_select <- function( - base, table_name, record_id = NULL, - fields = NULL, - filterByFormula = NULL, - maxRecord = NULL, - sort = NULL, - view = NULL, - pageSize = NULL, - offset = NULL, - combined_result = TRUE) { + base, table_name, record_id = NULL, + fields = NULL, + filterByFormula = NULL, + maxRecord = NULL, + sort = NULL, + view = NULL, + pageSize = NULL, + offset = NULL, + combined_result = TRUE) { search_path <- table_name if(!missing(record_id)) { @@ -449,10 +450,10 @@ air_make_request <- function(base, table_name, json_record_data, record_id = NUL # browser() res <- httr::PATCH(url = request_url, - httr::add_headers( - Authorization = paste("Bearer",air_api_key()), - 'Content-type' = "application/json"), - body = json_record_data) + httr::add_headers( + Authorization = paste("Bearer",air_api_key()), + 'Content-type' = "application/json"), + body = json_record_data) } if(method == "DELETE"){ @@ -644,15 +645,15 @@ air_table_funs <- function(base, table_name) { res_list <- list() res_list[["select"]] <- function( - record_id = NULL, - fields = NULL, - filterByFormula = NULL, - maxRecord = NULL, - sort = NULL, - view = NULL, - pageSize = NULL, - offset = NULL, - combined_result = TRUE + record_id = NULL, + fields = NULL, + filterByFormula = NULL, + maxRecord = NULL, + sort = NULL, + view = NULL, + pageSize = NULL, + offset = NULL, + combined_result = TRUE ){ air_select(base, table_name, record_id, fields, filterByFormula, maxRecord, sort, view, @@ -685,11 +686,11 @@ air_table_funs <- function(base, table_name) { res_list[["get"]] <- function( - record_id = NULL, - limit = NULL, offset = NULL, - view = NULL, - sortField = NULL, sortDirection = NULL, - combined_result = TRUE + record_id = NULL, + limit = NULL, offset = NULL, + view = NULL, + sortField = NULL, sortDirection = NULL, + combined_result = TRUE ){ air_get(base, table_name, record_id, limit, offset, view, sortField, sortDirection, combined_result) } From 37f8b6379126758a36caa46d6de48e31438dd154 Mon Sep 17 00:00:00 2001 From: Mindy Date: Thu, 24 Apr 2025 16:05:31 -0400 Subject: [PATCH 126/126] Update readme.md --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index c38637e..87cadf7 100644 --- a/readme.md +++ b/readme.md @@ -12,7 +12,7 @@ Provides access to the [Airtable API](http://airtable.com/api) ```r -devtools::install_github("ecohealthalliance/airtabler") +devtools::install_github("One-Health-Research-Consulting/airtabler") ``` ## Setup