diff --git a/.Rbuildignore b/.Rbuildignore index 9774842..df5700c 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -6,3 +6,5 @@ ^IMG$ ^readme_cache$ ^\.travis\.yml$ +^doc$ +^Meta$ diff --git a/.github/ISSUE_TEMPLATE/blank-issue.md b/.github/ISSUE_TEMPLATE/blank-issue.md new file mode 100644 index 0000000..a2a56c3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/blank-issue.md @@ -0,0 +1,10 @@ +--- +name: Blank Issue +about: Use this issue template for issues that are not bugs or enhancements +title: '' +labels: '' +assignees: '' + +--- + + diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..912f517 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,27 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Code to reproduce the behavior: + +**Expected behavior** +A clear and concise description of what you expected to happen. + + +**Session info (please complete the following information):** + +``` +sessionInfo() +``` + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..823ca61 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +**Describe why this feature should be added** +A clear and concise description of what the need is. + +**Describe the behavior you expect** +A clear and concise description of what you want to happen. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.gitignore b/.gitignore index bfd2ad0..bd39d79 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ .Rhistory .RData readme.html +testing_script.R +.env +/doc/ +/Meta/ diff --git a/DESCRIPTION b/DESCRIPTION index fe2dd0a..65bae38 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,17 +1,37 @@ Package: airtabler Type: Package Title: Interface to the Airtable API -Version: 0.1.6 -Date: 2017-09-17 +Version: 0.2.16 +Date: 2023-08-17 Author: Darko Bergant -Maintainer: Darko Bergant -Description: Provides access to the Airtable (airtable.com) API. +Maintainer: Collin Schwantes +Description: Fork from Darko Bergant's package. Provides access to the Airtable (airtable.com) API. Depends: R (>= 3.2.0) License: MIT + file LICENSE -URL: https://github.com/bergant/airtabler +URL: https://github.com/ecohealthalliance/airtabler LazyData: TRUE +Encoding: UTF-8 Imports: + glue, + dplyr, httr, - jsonlite -RoxygenNote: 6.0.1 + jsonlite, + readxl, + curl, + purrr, + utils, + snakecase, + tidyselect, + rlang, + stringr, + tibble, + deposits, + assertthat +Remotes: + ropenscilabs/deposits +RoxygenNote: 7.2.3 +Suggests: + knitr, + rmarkdown +VignetteBuilder: knitr diff --git a/NAMESPACE b/NAMESPACE index 96761eb..16d86ad 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,11 +1,51 @@ # Generated by roxygen2: do not edit by hand S3method(print,airtable.base) +export(air_create_description_table) +export(air_create_field) +export(air_create_metadata_table) +export(air_create_table) export(air_delete) +export(air_download_attachments) +export(air_dump) +export(air_dump_to_csv) +export(air_dump_to_json) +export(air_expand_csv_arrays) +export(air_fields_df_template) +export(air_fields_list_from_template) +export(air_generate_base_description) +export(air_generate_metadata_from_api) +export(air_generate_metadata_from_tables) export(air_get) +export(air_get_attachments) +export(air_get_base_description_from_table) +export(air_get_base_id_from_url) +export(air_get_id_from_url) +export(air_get_json) +export(air_get_metadata_from_table) +export(air_get_record_id_from_url) +export(air_get_schema) +export(air_get_table_id_from_url) +export(air_get_view_id_from_url) export(air_insert) +export(air_insert_data_frame) +export(air_list_bases) +export(air_make_json) +export(air_make_request) +export(air_post) export(air_select) +export(air_table_template) export(air_update) +export(air_update_data_frame) +export(air_update_description_table) +export(air_update_field) +export(air_update_metadata_table) export(airtable) +export(fetch_all) +export(fetch_all_json) +export(flatten_col_to_chr) export(get_offset) +export(get_unique_field_values) export(multiple) +export(read_excel_url) +export(set_diff) diff --git a/R/air_download_attachments.R b/R/air_download_attachments.R new file mode 100644 index 0000000..9a7757d --- /dev/null +++ b/R/air_download_attachments.R @@ -0,0 +1,141 @@ +#' Download Airtable file attachments +#' +#' Download an attachment stored in air tables. Returns original dataframe +#' with an additional field called attachment_file_paths. The attachment_file_paths +#' field is of class list so it can handle multiple attachments per record. File +#' paths are prepended with record ids so that all file names are unique. +#' +#' @param x Data frame. Output from air_get or fetch_all. +#' @param field String. Name of field with file attachments in base +#' @param dir_name String. Where should files be downloaded to? +#' Will create the folder if it does not exist. Folders created are recursively. +#' @param ... reserved for additional arguments. +#' @param include_attachment_id Logical. Should you include the airtable attachment +#' ID to guarantee all file names are unique? Default is true. +#' +#' @return Returns x with an additional field called attachment_file_paths +#' @export air_download_attachments +#' +#' @examples +#' \dontrun{ +#' +#' base <- "appXXXXXXXXX" +#' table_name <- "Table With Attachments" +#' +#' table_original <- air_get(base,table_name) +#' +#' table_with_file_paths <- air_download_attachments(x = table_with_attachments, +#' field = "attachment_field", +#' dir_name = "downloads") +#' +#' table_with_file_paths$attachment_file_paths +#' +#' } +#' +air_download_attachments <- function(x, field, dir_name = "downloads",include_attachment_id = TRUE,...){ + #browser() + + if(!is.data.frame(x)){ + rlang::abort("x is not a dataframe") + } + + if(!field %in% names(x)){ + error_msg <- glue::glue("{field} not found in names(x). Check the name of the column + used to store attachments in airtable") + + rlang::abort(error_msg) + } + + if(!is.list(x[,field])){ + error_msg <- glue::glue("{field} is not of class list. Verify the name of + the column used to store attachments in airtable") + rlang::warn(error_msg) + + field_file_paths <- sprintf("%s_file_paths",field) + + x$file_path <- NA + + # using dynamic names in case a base has multiple file attachment + # columns + x <- dplyr::rename(x,{{field_file_paths}} := file_path) + return(x) + } + + ### subset to necessary records ---- + + # get files + xfield <- purrr::pluck(x,field) + + ### get files ---- + dir.create(path = dir_name,recursive = TRUE) + + xlist <- purrr::map(xfield, function(x){ + + if(is.null(x$url)){ + ID <- x$id + warning(sprintf("Record ID %s is null",ID)) + return(NULL) + } + + # prepending attachment id in case the file naming convention + # of the user does not preclude duplicate file names for files + # with different contents - e.g. original file generation was + # structured like sample_1234/fasta.file sample_1235/fasta.file + + filename_dest <- x$filename + + if(include_attachment_id){ + filename_dest <- sprintf("%s_%s", x$id,filename_dest) + } else { + message("include_attachement_id = FALSE. If file names are repeated, + only the first file with that name will be downloaded.") + } + + dest <- sprintf("%s/%s", dir_name,filename_dest) + + # sometimes the same file is attached multiple times + # if the file is already downloaded, don't add it again + # each attachment gets a unique id, so if the file changes, + # that id changes + + + if(all(file.exists(dest))){ + not_downloaded_message <- glue::glue("\nFile already exists, not downloaded\n{dest}\n.") + print(not_downloaded_message) + return(dest) + } + + # wrap in a map so that it works on linux systems where urls must explicitly + # be a length one character vector + purrr::map2(x$url, dest, function(url_item, dest_item){ + if(is.character(url_item)& length(url_item )==1){ + a <- utils::download.file(url = url_item,destfile = dest_item) + print(a) + } else { + print("url item not character or length greater than one") + print(url_item) + } + }) + + + return(dest) + }) + + down_load_message <- glue::glue("Files downloaded here {dir_name}") + + message(down_load_message) + + field_file_paths <- sprintf("%s_file_paths",field) + + x$file_path <- xlist + + # using dynamic names in case a base has multiple file attachment + # columns + x <- dplyr::rename(x,{{field_file_paths}} := file_path) + return(x) + +} + + + + diff --git a/R/air_dump.R b/R/air_dump.R new file mode 100644 index 0000000..c54924f --- /dev/null +++ b/R/air_dump.R @@ -0,0 +1,1369 @@ +#' Get items that differ between x and y +#' +#' Unlike setdiff, this function creates the union of x and y then +#' removes values that are in the intersect, providing values +#' that are unique to X and values that are unique to Y. +#' +#' @param x a set of values. +#' @param y a set of values. +#' +#' @return Unique values from X and Y, NULL if no unique values. +#' @export +#' +#' @examples +#' a <- 1:3 +#' b <- 2:4 +#' +#' set_diff(a,b) +#' # returns 1,4 +#' +#' x <- 1:3 +#' y <- 1:3 +#' +#' set_diff(x,y) +#' # returns NULL +#' +set_diff <- function(x,y){ + u <- union(x,y) + i <- intersect(x,y) + j <- (u %in% i) + + if(all(j)){ + return(NULL) + } + + diff <- u[!(j)] + return(diff) +} + + +#' Create a new structural metadata table in the base +#' +#' @details Structural metadata describes the contents of your base and how they are linked. Structural +#' metadata can largely be derived from the base schema. +#' +#' @param base String. Base id +#' @param meta_data Data frame. Contains metadata records. From air_generate_metadata* +#' @param table_name String. name of the metadata table. default is "Meta Data" +#' @param field_descriptions Character vector. Descriptions of metadata table fields +#' @param type Character vector. Column types for metadata table fields. see https://airtable.com/developers/web/api/field-model +#' @param options Data frame. Options for fields in metadata table. +#' +#' @return List with outcome from creating the table and inserting the records +#' @export air_create_metadata_table +#' +#' @examples +#'\dontrun{ +#' # set base id +#' base <- "appXXXXXXXX" +#' # create metadata from api +#' metadata <- air_generate_metadata_from_api(base) +#' # add Meta Data table to base -- will not work if base already has a metadata +#' # table +#' log <- air_create_metadata_table(base,metadata) +#' +#'} +#' +#' +air_create_metadata_table <- function(base,meta_data,table_name = "Meta Data", field_descriptions = NA, + type = "singleLineText", options = NA){ + + # check for meta data table + ## if exists, stop + schema <- air_get_schema(base) + + if(table_name %in% schema$tables$name){ + msg <- glue::glue("{table_name} already exists in the base {base}. + Please use air_update_metadata_table to update the metadata table + or delete the table and re-run the function") + stop(msg) + } + + + # create fields_df + # add description for standard names + + + # + if(setequal(names(meta_data), c("field_name", "table_name", "field_desc", + "field_type", "field_id", "table_id", + "field_opts", "primary_key"))){ + + # create description object + field_descriptions <- c("https://schema.org/name", + "https://schema.org/name", + "https://schema.org/description", + "https://schema.org/category", + "https://schema.org/identifier", + "https://schema.org/identifier", + "https://schema.org/option", + "https://schema.org/Boolean" + ) + + + } + + fields_df <- air_fields_df_template(name = names(meta_data), + description = field_descriptions, + type = type, + options = options) + + # create list describing table + + table_list <- air_table_template(table_name = table_name, + description = "structural metadata for the base", + fields_df = fields_df) + # create table + + outcome_create_table <- air_create_table(base, table_list) + + # insert data + + outcome_insert_data <- tryCatch( + air_insert_data_frame(base = base,table_name = table_name,records = meta_data), + error=function(cond) { + + warning(cond) + + return("data not inserted") + }) + + if(is.character(outcome_insert_data)){ + stop("Table created but data not inserted. Check field types then use + air_insert_data_frame or air_update_metadata_table to add metadata + records.") + } + + + return(list("create_table" = outcome_create_table, + "insert_data" = outcome_insert_data)) + +} + +# add the description table to the base + +#' Create the descriptive metadata table for the base +#' +#' Descriptive metadata provides information about the base as a whole, who created it, +#' why, when, where can data be accessed, keywords, what license governs data use, etc. +#' Descriptive metadata facilitates data reuse by providing a point of contact for +#' future users, as well as attributes that allow the data to be entered into searchable +#' catalogs or archives. +#' +#' @details DCMI terms can be found here \url{https://www.dublincore.org/specifications/dublin-core/dcmi-terms/} +#' +#' @param base String. Base id +#' @param description Data frame. Description from air_get_base_description* or air_generate_base_description +#' @param table_name String. Name of description table +#' @param field_descriptions Character vector. Descriptions of metadata table fields. If NA, DCMI terms will be used where possible. +#' @param type Character vector. Column types for metadata table fields. see \url{https://airtable.com/developers/web/api/field-model} +#' @param options Data frame. Options for fields in metadata table. +#' +#' +#' @return List. Outputs from creating the table and inserting the records +#' @export air_create_description_table +#' +#' @examples +#'\dontrun{ +#' base = "appVjIfAo8AJlfTkx" +#' table_name= "description" +#' +#' description <- air_generate_base_description(title = "Example Base", +#' creator = "Collin Schwantes") +#' +#' air_create_description_table(base,description,table_name) +#'} +#' +air_create_description_table <- function(base, + description, + table_name = "Description", + field_descriptions = NA, + type = "singleLineText", + options = NA){ + + # check for description table + ## if exists, stop + schema <- air_get_schema(base) + + if(table_name %in% schema$tables$name){ + msg <- glue::glue("{table_name} already exists in the base {base}. + Please use air_update_description_table to update the description table + or delete the table and re-run the function") + stop(msg) + } + + + # create fields_df + # add description for dcmi terms + # check for dcmi terms + if(all(is.na(field_descriptions))){ + # create a placeholder dataframe + description_terms_df <- data.frame(col_names = names(description), dcmi_term = NA) + + # setup sprintf pattern + dcmi_uri <- "http://purl.org/dc/terms/%s" + + # check for a match for each dcmi term + description_terms_df_2 <- purrr::map_dfr(deposits::dcmi_terms(), function(dcmi_term){ + + # get index position for a term + dcmi_term_pos <- stringr::str_which(description_terms_df$col_names,pattern =dcmi_term,negate = FALSE) + + if(!rlang::is_empty(dcmi_term_pos)){ + + description_terms_df[dcmi_term_pos,"dcmi_term"] <- sprintf(dcmi_uri,dcmi_term) + + return(description_terms_df[dcmi_term_pos,]) + } + + return(NULL) + }) + + # get undescribed terms + + description_terms_df_3 <- dplyr::anti_join(description_terms_df,description_terms_df_2,"col_names") + + description_terms_df_4 <- rbind(description_terms_df_2,description_terms_df_3) + + description_terms_df_5 <- description_terms_df_4[match(names(description),description_terms_df_4$col_names),] + + field_descriptions <- description_terms_df_5$dcmi_term + + } + + fields_df <- air_fields_df_template(name = names(description), + description = field_descriptions, + type = type, + options = options) + + # create list describing table + + table_list <- air_table_template(table_name = table_name, + description = "descriptive metadata for the base", + fields_df = fields_df) + # create table + + outcome_create_table <- air_create_table(base, table_list) + + # insert data + + outcome_insert_data <- tryCatch( + air_insert_data_frame(base = base,table_name = table_name,records = description), + error=function(cond) { + + warning(cond) + + return("data not inserted") + }) + + if(is.character(outcome_insert_data)){ + stop("Table created but data not inserted. Check field types then use + air_insert_data_frame or air_update_description_table to add metadata + records.") + } + + + return(list("create_table" = outcome_create_table, + "insert_data" = outcome_insert_data)) + +} + + +#' Update the structural metadata table +#' +#' @param base String. Base id +#' @param meta_data Data frame. Contains metadata records. From air_generate_metadata* +#' @param table_name String. Name of metadata table +#' @param join_field String. Name of field to join new and current metadata. Likely \code{field_id} +#' @param record_id_field String. Name of record id field. Like \code{id} +#' +#' @return List. Log of results for updating metadata +#' @export air_update_metadata_table +#' +#' @examples +#'\dontrun{ +#' base = "appVjIfAo8AJlfTkx" +#' metadata <- air_generate_metadata_from_api(base = base) +#' air_update_metadata_table(base,metadata) +#'} +#' +air_update_metadata_table <- function(base,meta_data,table_name = "Meta Data", join_field = "field_id", record_id_field = "id"){ + + schema <- air_get_schema(base) + + # check for Meta Data table + check_for_md_table <- schema$tables$name %in% table_name + ## if no meta data table, stop + + + if(!any(check_for_md_table)){ + msg <- glue::glue("No table called {table_name} in base {base}. + Please use air_create_metadata_table to create the metadata table + or create it manually.") + stop(msg) + } + + ## get table_id + + table_id <- schema$tables[schema$tables$name == table_name,"id"] + + # pull down current meta data table + + current_metadata_table <- fetch_all(base,table_name) + + #create any new fields from meta_data + + message("creating log") + update_log <- list( + fields_created = NA, + records_updated = NA, + records_inserted = NA, + records_deleted = NA + ) + + + message("checking if any fields need to be added") + + # use schema in case table is empty + current_col_names <- schema$tables$fields[check_for_md_table][[1]]$name + col_check <- !names(meta_data) %in% current_col_names + + if(all(col_check)){ + msg <- glue::glue("The meta_data object and the metadata table in your base, {table_name}, share + no fields.") + warning(msg) + } + + if(any(col_check)){ + message("creating missing fields") + cols_to_create <- names(meta_data)[col_check] + + fields_created <- air_create_field(base = base, + table_id = table_id, + name = cols_to_create) + + message(fields_created) + + update_log$fields_created <- fields_created + + } + + # if the current metadata table is empty, then insert records + if(is.character(current_metadata_table)){ + message("added new records") + records_to_insert <- meta_data + + records_inserted <- air_insert_data_frame(base, table_name,records_to_insert) + + update_log$records_inserted <- records_inserted + + return(update_log) + } + + # compare with updated values + ## use field ids + message("checking which fields need to be updated") + ## assumes a certain structure for metadata + + min_update_df <- current_metadata_table[,c(join_field,record_id_field)] + + records_to_update <- dplyr::inner_join(meta_data,min_update_df,by = join_field ) + + # update records + message("updating records") + records_updated <- air_update_data_frame(base, table_name, records_to_update$id,records_to_update) + + update_log$records_updated <- records_updated + + # insert new records + message("added new records") + records_to_insert <- dplyr::anti_join(meta_data,min_update_df,by = join_field) + + records_inserted <- air_insert_data_frame(base, table_name,records_to_insert) + + update_log$records_inserted <- records_inserted + + + # drop records no longer in meta data + message("Checking for records no longer in the base") + + records_to_delete <- dplyr::anti_join(min_update_df,meta_data,by = join_field) + + + if(nrow(records_to_delete) >0){ + message("Deleting records no longer in the base") + records_deleted <- purrr::map(records_to_delete$id, function(id){ + air_delete(base, table_name,id) + }) + } else { + message("No Records deleted") + records_deleted <- "No records deleted" + } + + update_log$records_deleted <- records_deleted + + + return(update_log) +} + + +#' Update the description table +#' +#' Update the descriptive metadata table in airtable +#' +#' @param base String. Base id +#' @param description Data frame. Contains updated description +#' @param table_name String. Name of description table +#' @param join_field String. Field to perform join on +#' @param record_id_field String. Name of the record id field +#' +#' @return list that logs updates +#' @export air_update_description_table +#' +#' @examples +#' +#' \dontrun{ +#' +#' base <- "appXXXXXXXX" +#' table_name <- "Description" +#' # get description from table +#' description <- air_get_base_description_from_table(base, table_name) +#' # update the identifier field +#' description$identifier <- "fake.doi.xyz/029940" +#' # update the table +#' air_update_description_table(base,description) +#' +#' +#' } +#' +#' +air_update_description_table <- function(base,description, table_name = "Description", join_field = "title", record_id_field = "id"){ + + + # check for description + schema <- air_get_schema(base) + + # check for Description table + ## if no table, stop + + if(!table_name %in% schema$tables$name){ + msg <- glue::glue("No table called {table_name} in base {base}. + Please use air_create_description_table to create the descriptive + metadata table or create it manually.") + stop(msg) + } + + ## create empty update log list + + update_log <- list( + fields_created = NA, + records_updated = NA, + records_inserted = NA, + records_deleted = NA + ) + + + ## get table_id + + table_id <- schema$tables[schema$tables$name == table_name,"id"] + table_pos <- which(schema$tables$id == table_id) + # get column names from schema because empty columns are pulled + table_columns <- schema$tables$fields[[table_pos]]$name + ## check columns + col_check <- !names(description) %in% table_columns + + if(all(col_check)){ + msg <- glue::glue("The description object and the metadata table in your base, {table_name}, share + no fields.") + warning(msg) + } + #create any new fields from description + if(any(col_check)){ + cols_to_create <- names(description)[col_check] + + fields_created <- air_create_field(base = base, + table_id = table_id, + name = cols_to_create) + + message(fields_created) + + update_log$fields_created <- fields_created + + } + + # pull down current table + + current_metadata_table <- fetch_all(base,table_name) + # convert to tibble for more consistent behavior in joins + current_metadata_table<- tibble::as_tibble(current_metadata_table) + current_metadata_table <- current_metadata_table |> + dplyr::select(-createdTime) + + # compare with updated values + ## use field ids + ## assumes a certain structure for metadata + min_fields <- unique(join_field,record_id_field) + + min_update_df <- current_metadata_table[,min_fields] + + records_to_update <- dplyr::inner_join(description,min_update_df,by = join_field ) + + # update records + + records_updated <- air_update_data_frame(base, table_name, records_to_update$id,records_to_update) + + update_log$records_updated <- records_updated + + # insert new records -- each table describes a single base so there should be + # no records to insert + + records_to_insert <- dplyr::anti_join(description,min_update_df,by = join_field) + + if(nrow(records_to_insert) > 0){ + warning("Inserting a new record into the base. Check that the value in {join_field} matches + between current and updated description") + } + + records_inserted <- air_insert_data_frame(base, table_name,records_to_insert) + + update_log$records_inserted <- records_inserted + + + # drop records no longer in meta data + + records_to_delete <- dplyr::anti_join(min_update_df,description,by = join_field) + + + if(nrow(records_to_delete) >0){ + records_deleted <- purrr::map(records_to_delete$id, function(id){ + air_delete(base, table_name,id) + }) + } else { + records_deleted <- "No records deleted" + } + + update_log$records_deleted <- records_deleted + + + return(update_log) + +} + + +#' Pull the metadata table from Airtable +#' +#' Airtable allows all users to access the metadata API. +#' The recommended workflow for creating this table is to use +#' air_generate_metadata_from_api to extract the structural metadata from the base +#' schema and then use air_create_metadata_table to add the table to your +#' base. +#' +#' For information about creating metadata tables in your base see the +#' \href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} +#' +#' @details Requires the following fields: table_name, field_name +#' +#' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' +#' @param table_name String. Name of structural metadata table - the metadata that +#' describes how tables and fields fit together. +#' @param add_id_field Logical. If true, an "id" field is added to each table +#' @param field_names_to_snakecase Logical. If true, values in the field_names +#' column and the field in the metadata table themselves are are converted to snake_case +#' +#' @return data.frame with metadata table +#' @export air_get_metadata_from_table +#' +air_get_metadata_from_table <- function(base, table_name, add_id_field = FALSE, field_names_to_snakecase = TRUE){ + # get structural metadata table + str_metadata <- airtabler::fetch_all(base,table_name) + + # get original table names + str_md_names <- names(str_metadata) + + ## check for table_name, field_name + names(str_metadata) <- snakecase::to_snake_case(str_md_names) + + required_fields <- c("table_name","field_name") + if(!all(required_fields %in% names(str_metadata))){ + stop(glue::glue("metadata table must contain the + following fields: {required_fields}. Note + that field names are converted to snakecase + before check.")) + } + + ## make field names snake_case + if(field_names_to_snakecase){ + str_metadata$field_name <- snakecase::to_snake_case(str_metadata$field_name) + } + + ## add id field to all tables + if(add_id_field){ + + tables <- dplyr::distinct(.data = str_metadata,table_name,.keep_all = TRUE) + tables$field_desc <- "unique id assigned by airtable" + tables$field_type <- "singleLineText" + tables$field_id <- NA + tables$field_opts <- NA + + str_metadata <- rbind(str_metadata,tables) + + } + + if(!field_names_to_snakecase){ + names(str_metadata) <- str_md_names + } + + return(str_metadata) +} + + +#' Generate structural metadata from the api +#' +#' Structural metadata describes the contents of your base and how they are linked. +#' The structural metadata are created from the base schema. The nested schema +#' structure is flattened into a more user-friendly table which can then be +#' inserted as a table into the base with \code{air_created_metadata_table} and/or +#' used in a data export with \code{air_dump}. +#' +#' @details This function requires that the api token has the ability to read +#' the base schema. +#' +#' @param base String. Base id +#' @param metadata_table_name String. Name of exisiting structural metadata table if it exists +#' @param include_metadata_table Logical. Should the structural metadata table be included in the metadata? +#' @param field_names_to_snake_case Logical. Should the field names in the metadata table be snake_case? +#' +#' @return A data frame with metadata +#' @export air_generate_metadata_from_api +#' +#' @examples +#' +#' \dontrun{ +#' +#' base <- "appXXXXXXXX" +#' metadata <- air_generate_metadata_from_api(base) +#' +#' } + +air_generate_metadata_from_api <- function(base, + metadata_table_name = "Meta Data", + include_metadata_table = FALSE, + field_names_to_snake_case = TRUE){ + + # get base schema + schema <- air_get_schema(base) + + tables_df <- schema$tables + + if(!include_metadata_table){ + tables_df <- tables_df[stringr::str_detect(tables_df$name, + pattern = metadata_table_name, + negate = TRUE),] + } + + # parse base schema to populate metadata table + + #split by table id to parse with purrr + schema_list <- split(tables_df,f = tables_df$id) + + metadata_df <- purrr::map_dfr(schema_list, function(x){ + + # create metadata table skeleton + fields_df <-x$fields[[1]] + + fields_df$choices <- "" + fields_df$linkedTableID <- "" + + # get flattened choice names + if(!rlang::is_empty(fields_df$options$choices)){ + fields_df$choices <- purrr::map_chr(fields_df$options$choices,function(x){ + if(is.null(x)){ + return("") + } else { + return(paste(x$name,collapse = ", ")) + } + }) + } + + # get linked table id + if(!rlang::is_empty(fields_df$options$linkedTableId)){ + fields_df$linkedTableID <- fields_df$options$linkedTableId + } + + + fields_df <- fields_df |> + dplyr::mutate(field_opts = + dplyr::case_when( + type == "multipleSelects" | type == "singleSelect" ~ choices, + type == "multipleRecordLinks" ~ linkedTableID, + TRUE ~ "" + ) + ) + + + + + + # check that descriptions arent empty + + if(rlang::is_empty(fields_df$description)){ + fields_df$description <- "" + } + + md_df <- data.frame(field_name = fields_df$name, + table_name = x$name, + field_desc = fields_df$description, + field_type = fields_df$type, + field_id = fields_df$id, + table_id = x$id, + field_opts = fields_df$field_opts, + primary_key = as.character(x$primaryFieldId == fields_df$id) + ) + + if(!field_names_to_snake_case){ + names(md_df) <- c("Field Name","Table Name", "Field Desc","Field Type", + "Field ID", "Table ID", "Field Opts", "Primary Key") + } + + return(md_df) + }) + + return(metadata_df) +} + +##air_insert + +#' Generated Metadata from table names +#' +#' Deprecated: Use \code{air_generate_metadata_from_api} +#' +#' Generates a structural metadata table - the metadata that +#' describes how tables and fields fit together. Does not +#' include field types. +#' +#' @details For information about creating metadata tables in your base see the +#' \href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} +#' +#' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' +#' @param table_names Vector of strings. The names of your tables. eg c("table 1","table 2", etc.) +#' @param limit Number from 1-100. How many rows should we pull from each table to create the metdata? +#' Keep in mind that the airtable api will not return fields with "empty" values - "", false, or []. +#' Code runs faster if fewer rows are pulled. +#' +#' @return data.frame with structural metadata. +#' @export air_generate_metadata_from_tables + +air_generate_metadata_from_tables <- function(base, table_names,limit=1){ + warning('Deprecated: For more complete results, use air_generate_metadata_from_api. + Airtable does not return fields with empty values - "", false, or [].') + meta_data_table <- purrr::map_dfr(table_names,function(x){ + table_x <- airtabler::air_get(base,x,limit = limit ) + fields_x <- names(table_x) + + ## guess record types? + + md_df <- data.frame( field_name = fields_x, table_name = x, field_desc = "", field_type = "") + + return(md_df) + }) + + return(meta_data_table) +} + +#' Get base description from table +#' +#' Pull a table that has descriptive metadata. +#' Requires the following fields: +#' "title","primary_contact","email","description" +#' +#' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' +#' @param table_name String. Name of descriptive metadata table - the metadata that +#' describes the base and provides attribution +#' @param field_names_to_snakecase Logical. Should field names be converted to snakecase? +#' +#' @return data.frame with descriptive metadata. +#' @export air_get_base_description_from_table +#' +#' @examples +#' \dontrun{ +#' base <- "appXXXXXXXX" +#' table_name <- "Description" +#' air_get_base_description_from_table(base, table_name) +#' } +air_get_base_description_from_table<- function(base, table_name,field_names_to_snakecase = TRUE){ + #fetch table + desc_table <- airtabler::fetch_all(base,table_name) + # to snake case + if(field_names_to_snakecase){ + names(desc_table) <- snakecase::to_snake_case(names(desc_table)) + } + + required_fields <- c("title","primary_contact","email","description") + if(all(required_fields %in% names(desc_table))){ + return(desc_table) + } else { + + missing_rf <- required_fields[!required_fields %in% names(desc_table)] + + desc_table[missing_rf] <- "" + return(desc_table) + } + +} + +#' Generate descriptive metadata +#' +#' Creates a data.frame that describes the base. Descriptive metadata provides +#' information about the base as a whole: who created it, +#' why, when, where can data be accessed, keywords, what license governs data use, etc. +#' Descriptive metadata facilitates data reuse by providing a point of contact for +#' future users, as well as attributes that allow the data to be entered into searchable +#' catalogs or archives. +#' +#' @details See \href{https://www.dublincore.org/resources/userguide/creating_metadata/}{dublin core} for inspiration about additional attributes. +#' +#' @param title String. Title is a property that refers to the name or names by +#' which a resource is formally known. +#' @param creator String. Person or people who created the base +#' @param primary_contact String. Person or entity primarily responsible for +#' making the content of a resource +#' @param email String. Email of primary_contact +#' @param description String. This property refers to the description of +#' the content of a resource. The description is a potentially rich source of +#' indexable terms and assist the users in their selection of an appropriate +#' resource. +#' @param contributor String. An entity responsible for making contributions to the resource. +#' @param identifier String. An unambiguous reference to the resource within a given context. +#' @param license String. A legal document giving official permission to do something with the resource. "CC BY 4.0" +#' @param ... String. Additional descriptive metadata elements. See details. +#' Additional elements can be added as name pair values e.g. +#' \code{ isPartOf = "https://doi.org/00.00000/MyPaper01", isReferencedBy = "https://doi.org/10.48321/MyDMP01"} +#' @param created String. When was the base created? +#' +#' @return data.frame with descriptive metadata +#' @export +#' +#' @examples +#' +#' air_generate_base_description(title = "My Awesome Base" , +#' primary_contact= "Base Creator/Maintainer", +#' email = "email@@example.com", +#' base_description = "This base contains my awesome data +#' from a project studying XXX in YYY. Data in the base were collected +#' from 1900-01-01 to 1990-01-01 by researchers at Some Long Term Project.", +#' is_part_of = "https://doi.org/10.48321/MyDMP01", +#' isReferencedBy = "https://doi.org/10.5072/zenodo_sandbox.1062705" +#' ) +#' +air_generate_base_description <- function(title = NA, + creator= NA, + created=NA, + primary_contact=NA, + email = NA, + description = NA, + contributor = NA, + identifier =NA, + license = NA,...){ + desc_table <- tibble::tibble(title = title, + creator= creator, + created=created, + primary_contact=primary_contact, + email = email, + description = description, + contributor = contributor, + identifier =identifier, + license = license, + ...) + return(desc_table) +} + +### extract_base - returns a named list + +#' Dump all tables from a base into R +#' +#' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' +#' @param metadata Data.frame.Data frame with structural metadata - describes relationship between tables and fields. +#' Can be left as NULL if base already contains a table called meta data. +#' @param description Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. +#' Can be left as NULL if base already contains a table called description. +#' @param add_missing_fields Logical. Should fields described in the metadata data.frame be added to corresponding tables? +#' @param download_attachments Logical. Should attached files be downloaded? +#' @param ... Additional arguments to pass to air_download_attachments +#' @param attachment_fields Optional. character vector. +#' What field(s) should files be downloaded from? Default is to download all fields +#' with type multipleAttachments in metadata. +#' @param field_names_to_snakecase Logical. Should field names be +#' converted to snake case? +#' @param polite_downloads Logical. Use if downloading many files. Sets a delay +#' so that server is not overwhelmed by requests. +#' +#' @return List of data.frames. All tables from metadata plus the +#' description and metadata tables. +#' @export air_dump +#' +#' @note To facilitate joining on ids, see purrr::as_vector for converting list +#' type columns to vectors and +#' tidyr::unnest for expanding list columns. +#' +air_dump <- function(base, metadata= NULL, description = NULL, + add_missing_fields = TRUE, + download_attachments = TRUE, + attachment_fields=NULL, + polite_downloads = TRUE, + field_names_to_snakecase = TRUE, + ...){ + + # if metadata is null, check schema for metadata data table, + if(is.null(metadata)){ + message("No metadata provided. Metadata will be retrieved from the metadata + table or generated via API") + #get schema + base_schema <- air_get_schema(base) + # look for meta data table + table_names <- base_schema$tables$name + + metadata_check <- grepl("meta data",table_names,ignore.case = TRUE) + + if(any(metadata_check)){ + message("Retreiving metadata from table") + metadata <- fetch_all(base = base,table_name = table_names[metadata_check]) + } else { + message("Generating metadata from api with air_generate_metadta_from_api") + metadata <- air_generate_metadata_from_api(base = base) + } + } + + names(metadata) <- snakecase::to_snake_case(names(metadata)) + + ## check for required fields + required_fields <- c("table_name","field_name") + + if(!all(required_fields %in% names(metadata))){ + stop(glue::glue("metadata table must contain the + following fields: {required_fields}. Note + that field names are converted to snakecase + before check.")) + } + + + base_table_names <- unique(metadata$table_name) + + print(base_table_names) + table_list <- base_table_names |> + purrr::set_names() |> + purrr::map(function(x){ + + ## get fields from str_metadata + + fields_exp <- metadata[metadata$table_name == x,"field_name"] + + if(field_names_to_snakecase){ + fields_exp <- snakecase::to_snake_case(fields_exp) + } + + ## pull table - add check for blank tables + x_table <- airtabler::fetch_all(base,x) + + if(!is.data.frame(x_table)){ + x_table <- data.frame(id = character()) + } + + if(field_names_to_snakecase){ + names(x_table) <- snakecase::to_snake_case(names(x_table)) + } + + ## add in missing columns if any + fields_obs <- names(x_table) + + # check if any discrepancy between metadata and table + fields_diff <- set_diff(fields_exp,fields_obs) + #browser() + if(!is.null(fields_diff)){ + # check for fields in obs not in exp - error + obs_exp <- setdiff(fields_obs,fields_exp) + ignore_fields <- c("id","createdTime","created_time") + ignore_fields_pattern <- paste(ignore_fields,collapse = "|") + if(length(obs_exp) != 0 & !all(obs_exp %in% ignore_fields)){ + missing_fields <- obs_exp[!grepl(ignore_fields_pattern,obs_exp,ignore.case = FALSE)] + missing_fields_glue <- paste(missing_fields, collapse = "\n") + stop(glue::glue('The metadata table is missing the following fields from table {x}: + + {missing_fields_glue} + + Please update the metadata table via R with air_update_metadata_table + or manually at https://airtable.com/{base}')) + } + # check for fields in exp and not in obs - append unless frictionless + if(add_missing_fields){ + exp_obs <- setdiff(fields_exp,fields_obs) + x_table[exp_obs] <- list(character(0)) + } + } + + ## download files from attachment fields + + if(download_attachments){ + + if(rlang::is_empty(attachment_fields) & !"field_type"%in% names(metadata)){ + rlang::abort("Unclear which fields contain attachments. + Either use the attachment_fields argument or + supply a metadata dataframe with field_types == 'multipleAttachments' for + fields that should be downloaded") + } + + ## get attachment fields + if(is.null(attachment_fields)){ + metadata_table <- metadata[metadata$table_name == x,c("field_name","field_type")] + + attachment_fields <-metadata_table |> + dplyr::filter(field_type == "multipleAttachments") |> + dplyr::pull(field_name) + + if(rlang::is_empty(attachment_fields)){ + rlang::inform("No fields of type multipleAttachment. No files to download") + return(x_table) + } + } + + # check for field in table + if(is.character(attachment_fields)){ + if(!any(attachment_fields %in% names(x_table))){ + return(x_table) + } + } + + ## build up attachment fields on x_table + sleep_time <- 0 + if(polite_downloads){ + sleep_time <- 0.01 + } + for(af in attachment_fields){ + Sys.sleep(sleep_time) + x_table <- air_download_attachments(x_table,field = af,...) + } + } + + + return(x_table) + + }) + + table_list$metadata <- metadata + + #browser() + # check for description table + named_description <- grepl(pattern = "description",x = names(table_list), ignore.case = TRUE) + + + if(!is.null(description)){ + if(any(named_description)){ + warning("Base has a description table and a description data.frame was supplied to + this function. Inserting description data.frame at $description. Table + extract may be overwritten.") + } + table_list$description <- description + } else { + ### description may already be in the base, think about + ## how best to handle this + if( + all( + !named_description + ) + ){ + ## give null description + table_list$description <- air_generate_base_description() + } + } + + # named_description_post <- grepl(pattern = "description",x = names(table_list), ignore.case = TRUE) + # + # table_list[named_description_post][[1]]$created <- Sys.Date() + + return(table_list) +} + + +#' Flatten list columns to character +#' +#' Similar in spirit to purrr::flatten_chr except +#' that it can handle NULL values in lists and returns outputs +#' that can be written to csv. +#' +#' @details +#' Because the outputs are intended for use in CSV files, we must use +#' double quotes to indicate that the commas separating list values do +#' not delimit cells. This conforms to RFC 4180 standard for CSVs. +#' \url{https://datatracker.ietf.org/doc/html/rfc4180} +#' +#' +#' @param data_frame a data frame, tibble or other data frame like object +#' +#' @return data_frame with list columns converted to character vectors. +#' @export +#' +#' @examples +#' +#' data_frame <- data.frame(a = I(list(list("Hello"), +#' list("Aloha"), +#' NULL, +#' list("Hola","Bonjour","Merhaba") +#' )), +#' b = 1:4, +#' c = letters[1:4], +#' d = I(data.frame(id = 1:4, name = "bob", email = "bob@@example.com")) +#' ) +#' +#' test_df <- flatten_col_to_chr(data_frame) +#' +#' str(test_df) +#' +flatten_col_to_chr <- function(data_frame){ + for(i in names(data_frame)){ + #browser() + # get column values + col_from_df <- data_frame[[i]] + + if(is.list(col_from_df)){ + ## create an object to hold character values + chr_col <- as.character() + if(is.data.frame(col_from_df)){ + + n_r <- nrow(col_from_df) + + for(j in 1:n_r){ + list_element<- col_from_df[j,] + if(is.null(list_element)){ + list_element <- "" + } + + row_value<- sprintf('"%s"',paste(list_element,collapse = ",")) + + chr_col <- append(chr_col,row_value) + } + + } else { + n <- length(col_from_df) + + for(j in 1:n){ + list_element<- col_from_df[[j]] + if(is.null(list_element)){ + list_element <- "" + } + row_value<- sprintf('"%s"',paste(list_element,collapse = ",")) + + chr_col <- append(chr_col,row_value) + } + + } + + + data_frame[i] <- chr_col + } + } + return(data_frame) +} + + + +#' Save air_dump output to csv +#' +#' Saves data.frames from air_dump to csv files. File names are determined by +#' the names of the list objects from air_dump. Files will be saved in folder +#' with a unique name, inside the folder specified by \code{output_dir}. The +#' unique name is generated from a hash of the air_dump output. +#' +#' @param table_list List. List of data.frames output from \code{air_dump} +#' @param output_dir String. Folder containing output files +#' @param overwrite Logical. Should outputs be overwritten if they already exist? +#' @param attachments_dir String. What folder are base attachments stored in? +#' @param output_id String. Optional identifier for the data set - if NULL an +#' ID will be generated using a hash of the data. +#' @param names_to_snake_case Logical. Should field and table names be converted to snake_case? +#' +#' @return Vector of file paths +#' @export +air_dump_to_csv <- function(table_list,output_dir= "outputs", + attachments_dir=NULL, + overwrite = FALSE, + output_id = NULL, + names_to_snake_case = TRUE){ + + # create a unique id for the data + if(is.null(output_id)){ + output_id <- rlang::hash(table_list) + } + + + # check if data already exist + output_dir_path_final <- sprintf("%s/%s",output_dir,output_id) + if(dir.exists(output_dir_path_final) & !overwrite){ + message("data already exist, files not written. Set overwrite + to TRUE ") + return(list.files(output_dir_path_final,full.names = TRUE)) + } + + # create temp dir + temp_path <- tempdir() + output_dir_path <- sprintf("%s/%s",temp_path,output_id) + + ### consider using temp dir then copying once finished processing + + dir.create(output_dir_path,recursive = TRUE) + + purrr::walk2(table_list, names(table_list), function(x_table,y_table_name){ + + if(names_to_snake_case){ + ## clean table name + y_table_name_snake <- snakecase::to_snake_case(y_table_name) + ## clean up field names in table + x_names_snake <- snakecase::to_snake_case(names(x_table)) + + ## check that we didn't create duplicate field names + dup_check <- duplicated(names(x_table)) + + if(any(dup_check)){ + err_msg <- glue::glue("The following field names in table {y_table_name} are duplicated after converting to snake_case: + \n + {names(x_table)[dup_check]}\n + Fix field names in airtable or set `names_to_snake_case` to FALSE") + rlang::abort(err_msg) + } + + y_table_name <- y_table_name_snake + + names(x_table) <- x_names_snake + + } + x_table_flat <- flatten_col_to_chr(x_table) + + ## export to CSV + + output_file_path <- sprintf("%s/%s.csv",output_dir_path,y_table_name) + + utils::write.csv(x_table_flat,output_file_path,row.names = FALSE) + }) + + ## copy from temp to final + dir.create(output_dir_path_final,recursive = TRUE) + outputs_list <- list.files(output_dir_path,full.names = T) + + file.copy(from = outputs_list,to = output_dir_path_final,recursive = FALSE ,copy.mode = TRUE) + + ## copy attachments into folder + if(!is.null(attachments_dir)){ + message("copying attachments") + file.copy(from = attachments_dir, to = output_dir_path_final,recursive = TRUE ,copy.mode = TRUE ) + } + return(list.files(output_dir_path_final,full.names = TRUE)) + +} + + +### extract_base - returns a named list + +#' Dump all tables from a base into json files +#' +#' Essentially air_get without converting to Rs. Does not add fields with empty +#' values. +#' +#' @param base String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX' +#' @param metadata Data.frame.Data frame with structural metadata - describes relationship between tables and fields. +#' @param description Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. +#' Can be left as NULL if base already contains a table called description +#' @param output_dir String. Where should json files be saved? +#' @param overwrite Logical. If data are not unique, should files be overwritten? +#' +#' @return List of data.frames. All tables from metadata plus the +#' description and metadata tables. +#' @export air_dump_to_json +#' +air_dump_to_json <- function(base, metadata, description = NULL, output_dir= "outputs", overwrite = FALSE){ + + names(metadata) <- snakecase::to_snake_case(names(metadata)) + + ## check for required fields + required_fields <- c("table_name") + + if(!all(required_fields %in% names(metadata))){ + stop(glue::glue("metadata table must contain the + following field: {required_fields}. Note + that field names are converted to snakecase + before check.")) + } + + + base_table_names <- unique(metadata$table_name) + + print(base_table_names) + + json_list <- base_table_names |> + purrr::set_names() |> + purrr::map(function(x){ + + ### no expected fields, just json + ### see air_make_json for refactor + + x_json <- fetch_all_json(base,x) + + return(x_json) + + }) + + json_list$metadata <- jsonlite::toJSON(metadata) + + # check for description table + named_description <- grepl(pattern = "description",x = names(json_list), ignore.case = TRUE) + + if(!is.null(description)){ + if(any(named_description)){ + warning("Base has a description table and a description data.frame was supplied to + this function. Inserting description data.frame at $description. Table + extract may be overwritten.") + } + json_list$description <- jsonlite::toJSON(description) + } + + ## does not add a description table if not present + + + ## write to files + output_id <- rlang::hash(json_list) + + output_dir_path <- sprintf("%s/%s",output_dir,output_id) + + dir.create(output_dir_path,recursive = TRUE) + + purrr::walk2(json_list, names(json_list), function(x_table,y_table_name){ + + output_file_path <- sprintf("%s/%s.json",output_dir_path,y_table_name) + + jsonlite::write_json(x_table,path = output_file_path) + }) + + return(list.files(output_dir_path,full.names = TRUE)) + + + +} + + + +### write to db + +### recover from metadata - JS code to regenerate tables +# +# air_js_for_tables <- function(metadata){ +# names(metadata) <- snakecase::to_snake_case(names(metadata)) +# table_names <- unique(metadata$table_name) +# +# ## not sufficient for all field types, need to think more deeply about this +# purrr::map(table_names, function(x){ +# field_names <- metadata[metadata$table_name == x,"field_name"] +# field_types <- metadata[metadata$table_name == x,"field_type"] +# create_field <- sprintf('{name: "%s", type: "%s"}',field_name,field_type) +# }) +# +# } + + + diff --git a/R/air_expand_csv_arrays.R b/R/air_expand_csv_arrays.R new file mode 100644 index 0000000..61e6aaa --- /dev/null +++ b/R/air_expand_csv_arrays.R @@ -0,0 +1,51 @@ +#' Expand arrays stored in CSVs +#' +#' This function helps users work with airtable data that has been exported to CSVs. +#' Because airtable uses nested data structures (json arrays), the data must be +#' flattened to be stored in a csv. The standard way to store arrays in a csv +#' is to wrap the array in quotes and separate each item with commas. So an array +#' stored in a csv would look like "item 1,item 2,...,item n". This function will +#' convert arrays stored in csvs to either a list or a vector and removes the +#' surrounding quotes. +#' +#' +#' @param x Character. likely a vector or field in a dataframe. +#' @param simplify_to_vector Logical. Should expanded arrays be converted from +#' lists to vectors? For lists with multiple elements at a given position, the length of the output +#' may be greater than the length of the input. See [tidyr::unnest()] for expanding +#' list columns. +#' +#' +#' @return A vector or list of expanded arrays. +#' @export air_expand_csv_arrays +#' +#' @examples +#' +#' +#' # example vector data +#' x <- c("item 1,item 2,item 3","apple,orange,banana","1,2,3","") +#' +#' # to list +#' air_expand_csv_arrays(x) +#' +#' # to vector +#' air_expand_csv_arrays(x,simplify_to_vector = TRUE) +#' +#' +air_expand_csv_arrays <- function(x,simplify_to_vector = FALSE){ + + if(!is.character(x)){ + rlang::abort("x must be class character. x is class {class(x)}") + } + + split_x <- stringr::str_split(x,",") + + x_out <- purrr::map(.x = split_x,.f = stringr::str_replace_all,pattern = "^\"|\"$",replacement = "") + + if(simplify_to_vector){ + x_out <- purrr::as_vector(x_out,.type = "character") + } + + return(x_out) + +} diff --git a/R/air_get_attachments.R b/R/air_get_attachments.R new file mode 100644 index 0000000..705fd8f --- /dev/null +++ b/R/air_get_attachments.R @@ -0,0 +1,77 @@ +#' Get Airtable file attachments +#' +#' Extract the contents of an attachment stored in Airtable. Currently only setup +#' to work with Excel files. Planned expansion to other file types. +#' For excel files, returns a named list. +#' +#' @seealso \code{air_download_attachments} +#' +#' @param base String. ID for the base or app to be fetched +#' @param table_name String. Name of the table to be fetched from the base +#' @param field String. Name of field with file attachments in base +#' @param extract_type String. File type to be extracted. +#' Should be one of: excel +#' @param extract_field String. Name of extract field that will be created +#' @param ... Additional arguments to pass to \code{air_get} +#' @param download_file Logical. Should files be downloaded? +#' @param include_attachment_id Logical. Should the attachment ID be included in the file name? +#' Default is true to ensure unique file names. +#' @param dir_name String. Where should files be downloaded to? +#' Will create the folder if it does not exist. +#' @param skip Numeric. How many lines should be skipped? See \code{readxl::read_excel} skip. +#' @param parse_all_sheets Logical. Should all sheets in spreadsheet be parsed? +#' +#' @return named list of data frames +#' @export air_get_attachments +#' +#' @examples +#' +#' \dontrun{ +#' +#' base <- "appXXXXXXXXX" +#' table_name <- "table with excel attachments" +#' +#' table_with_attachments <- air_get_attachments(base,table_name, field = "attachment_field" ) +#' +#' } +#' +#' +air_get_attachments <- function(base, table_name, field, download_file = FALSE,include_attachment_id = TRUE, dir_name = "downloads", extract_type ="excel", extract_field ="excel_extract", skip = 0, parse_all_sheets = FALSE, ...){ + + # get data + x <- fetch_all(base,table_name,...) + + ### subset to necessary records ---- + + # get files + xfield <- purrr::pluck(x,field) + + ### get files ---- + + if(download_file){ + x <- air_download_attachments(x,field = field,dir_name = dir_name,include_attachment_id ) + } + + ### extract excel ---- + + if(extract_type == "excel"){ + + xlist <- purrr::map(xfield,function(x){ + if(is.null(x$url)){ + ID <- x$id + warning(sprintf("Record ID %s is null",ID)) + return(NULL) + } + read_excel_url(x$url, skip = skip,parse_all_sheets = parse_all_sheets) ## need to be able to pass additional arguments + }) + + ## add extract to data frame ---- + x[[extract_field]] <- xlist + } + + return(x) +} + + + + diff --git a/R/air_get_json.R b/R/air_get_json.R new file mode 100644 index 0000000..2dc339e --- /dev/null +++ b/R/air_get_json.R @@ -0,0 +1,129 @@ +#' Get a list of records or retrieve a single record as JSON +#' +#' Returns JSON objects from GET requests +#' +#' @param base Airtable base +#' @param table_name Table name +#' @param record_id (optional) Use record ID argument to retrieve an existing +#' record details +#' @param limit (optional) A limit on the number of records to be returned. +#' Limit can range between 1 and 100. +#' @param offset (optional) Page offset returned by the previous list-records +#' call. Note that this is represented by a record ID, not a numerical offset. +#' @param view (optional) The name or ID of the view +#' @param sortField (optional) The field name to use for sorting +#' @param sortDirection (optional) "asc" or "desc". The sort order in which the +#' records will be returned. Defaults to asc. +#' @param combined_result If TRUE (default) all data is returned in the same data. +#' If FALSE table fields are returned in separate \code{fields} element. +#' @param fields (optional) Only data for fields whose names are in this list +#' will be included in the records. If you don't need every field, you can use +#' @param pretty Logical. Should JSON be returned in human readable form? +#' this parameter to reduce the amount of data transferred. +#' @return A data frame with records or a list with record details if +#' \code{record_id} is specified. +#' +#' @export air_get_json +air_get_json <- function(base, table_name, + record_id = NULL, + limit = NULL, + offset = NULL, + view = NULL, + fields = NULL, + sortField = NULL, + sortDirection = NULL, + combined_result = TRUE, + pretty = FALSE) { + + search_path <- table_name + + if(!missing(record_id)) { + search_path <- paste0(search_path, "/", record_id) + } + request_url <- sprintf("%s/%s/%s?", air_url, base, search_path) + request_url <- utils::URLencode(request_url) + + # append parameters to URL: + param_list <- as.list(environment())[c( + "limit", "offset", "view", "sortField", "sortDirection")] + param_list <- param_list[!sapply(param_list, is.null)] + if(!is.null(fields)) { + param_list <- c(param_list, list_params(x = fields, par_name = "fields")) + } + + request_url <- httr::modify_url(request_url, query = param_list) + request_url <- gsub(pattern = "fields=",replacement = "fields%5B%5D=",x = request_url) + + # call service: + res <- httr::GET( + url = request_url, + config = httr::add_headers(Authorization = paste("Bearer", air_api_key())) + ) + air_validate(res) # throws exception (stop) if error + ret <- httr::content(res, as = "text") #returns json object + if(pretty){ + ret <- jsonlite::prettify(ret) + } + return(ret) + +} + + +#' Get the full outputs of a table as single json object +#' +#' @param base String. Base ID +#' @param table_name String. Table name +#' @param ... additional parameters to pass to air_get_json +#' +#' @return json as string +#' @export fetch_all_json +#' +#' @examples +#' +#' \dontrun{ +#' base <- "appXXXXXXX" +#' table_name <- "My Table" +#' +#' fetch_all_json(base, table_name) +#' +#' } +#' +fetch_all_json <- function(base, table_name, ...) { + out <- list() + out[[1]] <- air_get_json(base, table_name, combined_result = FALSE,...) + if(length(out[[1]]) == 0){ + emptyTableMessage <- glue::glue("The queried view for {table_name} in {base} is empty") + warning(emptyTableMessage) + return(emptyTableMessage) + } else { + offset <- airtabler::get_offset(air_parse(out[[1]])) #parse out offset + while (!is.null(offset)) { + json_text <- list(air_get_json(base, table_name, combined_result = FALSE, offset = offset, ...)) + out <- c(out, json_text) + offset <- airtabler::get_offset(air_parse(out[[length(out)]])) + } + + ## map over json text and remove offset attribute + out_json <- purrr::map_chr(out, function(x){ + + x_drop_offset <- gsub( '\\],\"offset\\":\\"\\w{1,}/\\w{1,}\\"}$',"",x) + + }) + + # drop initial json section in all but first record + out_json_2 <- out_json + + for(i in 1:length(out_json)){ + if(i > 1){ + out_json_2[i] <- gsub("\\{\"records\"\\:\\[",",",out_json[i]) + } + next() + } + + # make single json string + json_delimited <- paste(out_json_2,collapse = "") + + return(json_delimited) + + } +} diff --git a/R/air_metadata_api.R b/R/air_metadata_api.R new file mode 100644 index 0000000..8c703ed --- /dev/null +++ b/R/air_metadata_api.R @@ -0,0 +1,628 @@ +#' Get base schema +#' +#' Get the schema for the tables in a base. This is a wrapper for the api call +#' Get base schema. +#' +#' @section Using Metadata API: +#' Metadata api is currently available to all users. +#' +#' @param base String. Airtable base ID +#' @param ... reserved for additional parameters +#' +#' @return list of schema +#' @export air_get_schema + +air_get_schema <- function(base, ...){ + request_url <- sprintf("%s/%s/tables", air_meta_url, base) + request_url <- utils::URLencode(request_url) + + # call service: + res <- httr::GET( + request_url, + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ) + ) + + air_validate(res) + # may need a new air_parse function + + res_content <- httr::content(res,as = "text") + + schema <- jsonlite::fromJSON(res_content) + + return(schema) +} + +type_option_map <-function(){ + + # will be used to validate options provided to different field types + "https://airtable.com/developers/web/api/field-model" + + # time options are deeply nested + NULL + + +} + +#' Template for for creating a table from a tibble +#' +#' Convenience function for creating the content of tables that will created or +#' updated viaAPI. +#' +#' @param name String. Names of fields in the table +#' @param description String. Descriptions of fields +#' @param type String. Type of columns. For values see \url{https://airtable.com/developers/web/api/model/field-type} +#' @param options List. Options will be converted from lists to JSON. For field options see \url{https://airtable.com/developers/web/api/field-model} +#' +#' @return Tibble with attributes required for fields in a table +#' @export air_fields_df_template +#' +#' @examples +#' +#'\dontrun{ +#' base <- "appQ94sELAtFnXPxx" +#' +#' base_schema <- air_get_schema(base) +#' +#' tables<- base_schema$tables +#' +#' field_names <- c("Planet","Chapter","Book", "Known Inhabitants") +#' +#' field_desc <- c("Name of planet in Foundation Series", +#' "Chapters where planet is referenced", +#' "Books where planet is referenced", +#' "Characters mentioned as living on or being from that planet") +#' +#' field_types <- c("singleLineText",rep("multipleRecordLinks",3)) +#' +#' field_options <- c(NA,list( +#' list( +#' linkedTableId = tables[tables$name == "Chapter","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Book","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Character","id"] +#' ) +#' ) +#' ) +#' +#' field_df<- air_fields_df_template(name = field_names, +#' description = field_desc, +#' type = field_types, +#' options = field_options) +#' +#' table_list <- air_table_template(table_name = "Planet", +#' description = "Planets of Foundation", +#' fields_df = field_tables) +#' +#' air_create_table(base, table_list) +#'} +air_fields_df_template <- function(name,description, type, options = NA){ + df <- tibble::tibble(name = name, + description = description, + type = type, + options = options) + + return(df) +} + +#' Convert field data frame to list +#' +#' Converts the field data frame to a list of easier translation to JSON +#' +#' @param df Data frame. From air_fields_df_template +#' +#' @return List. Structured for easy parsing into JSON +#' @export air_fields_list_from_template +#' +#' @examples +#'\dontrun{ +#' base <- "appQ94sELAtFnXPxx" +#' +#' base_schema <- air_get_schema(base) +#' +#' tables<- base_schema$tables +#' +#' field_names <- c("Planet","Chapter","Book", "Known Inhabitants") +#' +#' field_desc <- c("Name of planet in Foundation Series", +#' "Chapters where planet is referenced", +#' "Books where planet is referenced", +#' "Characters mentioned as living on or being from that planet") +#' +#' field_types <- c("singleLineText",rep("multipleRecordLinks",3)) +#' +#' field_options <- c(NA,list( +#' list( +#' linkedTableId = tables[tables$name == "Chapter","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Book","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Character","id"] +#' ) +#' ) +#' ) +#' +#' field_df <- air_fields_df_template(name = field_names, +#' description = field_desc, +#' type = field_types, +#' options = field_options) +#' +#' fields_list <- air_fields_list_from_template(df = fields_df) +#' +#'} +air_fields_list_from_template <- function(df){ + ## create a list of field objects + purrr::pmap(df, function(name, + type, + description, + options){ + + field_list <- list(name = name, + type = type) + + if(!is.na(description)){ + field_list$description <- description + } + + if(!all(is.na(options))){ + field_list$options <- options + } + return(field_list) + }) + +} + +#' Template for lists that describe tables in Airtable +#' +#' @param table_name String. Name of table +#' @param description String. Description of the table +#' @param fields_df Data frame. Data frame describing the field in a table. +#' Should contain a name, description,type, and options field. +#' if +#' +#' @return List with table name, description, and fields +#' @export air_table_template +#' +#' @examples +#' +#' \dontrun{ +#' base <- "appQ94sELAtFnXPxx" +#' +#' base_schema <- air_get_schema(base) +#' +#' tables<- base_schema$tables +#' +#' field_names <- c("Planet","Chapter","Book", "Known Inhabitants") +#' +#' field_desc <- c("Name of planet in Foundation Series", +#' "Chapters where planet is referenced", +#' "Books where planet is referenced", +#' "Characters mentioned as living on or being from that planet") +#' +#' field_types <- c("singleLineText",rep("multipleRecordLinks",3)) +#' +#' field_options <- c(NA,list( +#' list( +#' linkedTableId = tables[tables$name == "Chapter","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Book","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Character","id"] +#' ) +#' ) +#' ) +#' +#' field_df<- air_fields_df_template(name = field_names, +#' description = field_desc, +#' type = field_types, +#' options = field_options) +#' +#' table_list <- air_table_template(table_name = "Planet", +#' description = "Planets of Foundation", +#' fields_df = field_tables) +#' +#' air_create_table(base, table_list) +#'} +#' +#' +air_table_template <- function(table_name, description, fields_df ){ + + valid_cols <- c("description","name","type","options") + + invalid_names_check <- !names(fields_df)%in%valid_cols + + if(any(invalid_names_check)){ + + msg <- glue::glue("Invalid column name in field_df: {names(fields_df)[invalid_names_check]} + valid column names are: {valid_cols}") + stop(msg) + } + + # check for required columns + required_cols <- valid_cols[1:3] + + required_names_check <- !required_cols %in% names(fields_df) + + if(any(required_names_check)){ + + msg <- glue::glue("Missing required column name in fields_df: {required_cols[required_names_check]} + required column names are: {required_cols}") + stop(msg) + } + + ## check that types have necessary options - TODO + + ## check that options have appropriate values - TODO + + ## convert data frame to list of field objects for easier translation to JSON + + fields_list <- air_fields_list_from_template(df = fields_df) + + ## create output + table_list <- list( + "name" = table_name, + "description" = description, + "fields" = fields_list + ) + + return(table_list) +} + +#' A function to create new tables in a base +#' +#' Takes a list object with appropriate arguments (see \code{air_table_template}) +#' converts it to JSON then adds it to the specified base. +#' +#' @note See https://airtable.com/developers/web/api/create-table +#' @param base String. ID for the base +#' @param table_list List. see \code{air_table_template} +#' +#' @return Data frame of table schema +#' @export air_create_table +#' +#' @examples +#' +#' \dontrun{ +#' base <- "appQ94sELAtFnXPxx" +#' +#' base_schema <- air_get_schema(base) +#' +#' tables<- base_schema$tables +#' +#' field_names <- c("Planet","Chapter","Book", "Known Inhabitants") +#' +#' field_desc <- c("Name of planet in Foundation Series", +#' "Chapters where planet is referenced", +#' "Books where planet is referenced", +#' "Characters mentioned as living on or being from that planet") +#' +#' field_types <- c("singleLineText",rep("multipleRecordLinks",3)) +#' +#' field_options <- c(NA,list( +#' list( +#' linkedTableId = tables[tables$name == "Chapter","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Book","id"] +#' ) +#' ), +#' list( +#' list( +#' linkedTableId = tables[tables$name == "Character","id"] +#' ) +#' ) +#' ) +#' +#' field_df<- air_fields_df_template(name = field_names, +#' description = field_desc, +#' type = field_types, +#' options = field_options) +#' +#' table_list <- air_table_template(table_name = "Planet", +#' description = "Planets of Foundation", +#' fields_df = field_df) +#' +#' air_create_table(base, table_list) +#' +#' } +air_create_table <- function(base, table_list){ + request_url <- sprintf("%s/%s/tables", air_meta_url, base) + request_url <- utils::URLencode(request_url) + + fields_json <- jsonlite::toJSON(table_list,pretty = TRUE,auto_unbox = TRUE) + + # call service: + res <- httr::POST( + request_url, + httr::content_type("application/json"), + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ), + body = fields_json + ) + + air_validate(res) + # may need a new air_parse function + + res_content <- httr::content(res,as = "text") + + schema <- jsonlite::fromJSON(res_content) + + return(schema) +} + +#' Create a new field in a table +#' +#' See https://airtable.com/developers/web/api/create-field +#' +#' @param base String. Base id +#' @param table_id String. Table id. Can be found using \code{air_get_schema} +#' @param name String. Name of the field +#' @param description String. Description of the field +#' @param type String. Type of field. See https://airtable.com/developers/web/api/field-model +#' @param options Data frame. See https://airtable.com/developers/web/api/field-model +#' +#' @return description of newly created field as a list +#' @export air_create_field +#' +#' @examples +#' \dontrun{ +#' base_schema <- air_get_schema(base) +#' +#' base_schema$tables +#' +#' air_create_field(base,table_id = base_schema$tables$id[[4]], +#' name = "Has Nucleics", +#' description = "Logical. Does this planet have nucleics?", +#' type = "checkbox", +#' options = list( +#' list( +#' "color"= "greenBright", +#' "icon"= "check" +#' ) +#' ) +#' ) +#' } +#' +air_create_field <- function(base, + table_id, + name, + description = NA, + type = "singleLineText", + options= NA){ + + field_df <- air_fields_df_template(name = name,description = description, type = type, options = options) + + + fields_list <- air_fields_list_from_template(field_df) + + "https://api.airtable.com/v0/meta/bases/{baseId}/tables/{tableId}/fields" + + request_url <- sprintf("%s/%s/tables/%s/fields", air_meta_url, base,table_id) + request_url <- utils::URLencode(request_url) + + ## fields must be created one at a time + schema_list <- purrr::map(fields_list,function(field_item){ + fields_json <- jsonlite::toJSON(field_item,pretty = TRUE,auto_unbox = TRUE) + + # call service: + res <- httr::POST( + request_url, + httr::content_type("application/json"), + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ), + body = fields_json + ) + + air_validate(res) + # may need a new air_parse function + + res_content <- httr::content(res,as = "text") + + schema <- jsonlite::fromJSON(res_content) + + return(schema) + }) + + return(schema_list) + +} + + +#' Update a field name and/or description +#' +#' Must update either the name or the description. +#' See "https://airtable.com/developers/web/api/update-field" for more details. +#' +#' @param base String. Base id +#' @param table_id String. ID for table that contains the field to be updated +#' @param field_id String. ID of field to be updated +#' @param name String. updated name (optional) +#' @param description String. updated description (option) +#' +#' @return List. Describes the changes that happened to the field +#' @export air_update_field +#' +#' @examples +#' \dontrun{ +#' base <- "appVjIfAo8AJlfTkx" +#' +#' schema <- air_get_schema("appVjIfAo8AJlfTkx") +#' +#' table_id <- schema$tables[1,c("id")] +#' +#' field_id <- schema$tables$fields[[1]][2,]$id +#' +#' ## update name and description +#' +#' name <- "New Name" +#' +#' description <- "Updated Description" +#' +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +#' name = name, description = description) +#' +#' ### just name +#' +#' name <- "New New Name" +#' +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +#' name = name) +#' +#' +#' ## just description +#' +#' description <- "Better description" +#' +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +#' description = description) +#' +#' ## set name to number +#' +#' name <- 1234 +#' +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +#' name = name) +#' +#' +#' # set description to number +#' +#' description <- 1234 +#' +#' out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +#' description = description) +#' +#' # bulk update names and descriptions from a data frame +#' +#' field_ids <- schema$tables$fields[[1]]$id +#' +#' field_names <- sprintf("%s_bulk_update",schema$tables$fields[[1]]$name) +#' +#' field_descriptions <- sprintf("%s BULK UPDATE", +#' schema$tables$fields[[1]]$description) +#' +#' df <- data.frame("field_id"= field_ids,"name"=field_names, +#' "description"=field_descriptions) +#' +#' purrr::pmap(df,function(field_id,name,description){ +#' air_update_field(base = base,table_id = table_id,field_id = field_id, +#' name = name, description = description) +#' }) +#' } +air_update_field<- function(base, table_id, field_id, name = NULL, description = NULL){ + + if(rlang::is_empty(name) & rlang::is_empty(description)){ + rlang::abort("Must provide either a name or description") + } + + ## create the field list + field_list<- list() + + # add name property + if(!rlang::is_empty(name)){ + assertthat::assert_that(is.character(name),length(name) ==1) + field_list$name <- name + } + + # add description property + if(!rlang::is_empty(description)){ + assertthat::assert_that(is.character(description),length(description) ==1) + field_list$description <- description + } + + assertthat::assert_that(length(field_list) > 0) + + # create + "https://api.airtable.com/v0/meta/bases/{baseId}/tables/{tableId}/fields/{columnId}" + + request_url <- sprintf("%s/%s/tables/%s/fields/%s", air_meta_url, base,table_id,field_id) + request_url <- utils::URLencode(request_url) + + + ## fields must be updated one at a time + fields_json <- jsonlite::toJSON(field_list,pretty = TRUE,auto_unbox = TRUE) + + # call service: + res <- httr::PATCH( + request_url, + httr::content_type("application/json"), + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ), + body = fields_json + ) + + air_validate(res) + # may need a new air_parse function + + res_content <- httr::content(res,as = "text") + + schema <- jsonlite::fromJSON(res_content) + + return(schema) +} + +#' Get list of bases for an Token +#' +#' Each token you provisision is given access to a certain set of bases or +#' workspaces. This function lists all bases associated with a token. +#' +#' @param request_url String. URL for api endpoint +#' +#' @return list. List of bases a token can access. +#' @export air_list_bases +#' +#' +#' @examples +#' +#' \dontrun{ +#' air_list_bases() +#' } +#' +air_list_bases <- function(request_url = "https://api.airtable.com/v0/meta/bases"){ + + request_url <- utils::URLencode(request_url) + + # call service: + res <- httr::GET( + request_url, + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ) + ) + + air_validate(res) + # may need a new air_parse function + + res_content <- httr::content(res,as = "text") + + base_list <- jsonlite::fromJSON(res_content) + + return(base_list) +} + + + diff --git a/R/air_post.R b/R/air_post.R new file mode 100644 index 0000000..163f98b --- /dev/null +++ b/R/air_post.R @@ -0,0 +1,96 @@ +#' Get a list of records +#' +#' Retrieve records where the request url would be over 16k characters (e.g. +#' complicated formula) or has more than 21 fields listed in the request. +#' +#' You can retrieve records in an order of a view by providing the name or ID of +#' the view in the view query parameter. The results will include only records +#' visible in the order they are displayed. +#' +#' @seealso [air_get()] +#' +#' @param base String. Airtable base +#' @param table_name String. Table name +#' @param limit Numeric. (optional) A limit on the number of records to be returned. +#' Limit can range between 1 and 100. +#' @param offset Numeric. (optional) Page offset returned by the previous list-records +#' call. Note that this is represented by a record ID, not a numerical offset. +#' @param view String. (optional) The name or ID of the view +#' @param sortField String. (optional) The field name to use for sorting +#' @param sortDirection String. (optional) "asc" or "desc". The sort order in which the +#' records will be returned. Defaults to asc. +#' @param combined_result Logical. If TRUE (default) all data is returned in the same data. +#' If FALSE table fields are returned in separate \code{fields} element. +#' @param fields List. (optional) Only data for fields whose names are in this list +#' will be included in the records. Does not work when retrieving individual records with \code{record_id} +#' @param filterByFormula String. Use a formula to filter results. See \href{airtable docs}{https://support.airtable.com/hc/en-us/articles/223247187-How-to-sort-filter-or-retrieve-ordered-records-in-the-API} +#' this parameter to reduce the amount of data transferred. +#' @return A data frame with records +#' @export + +air_post <- function(base, + table_name, + limit = NULL, + offset = NULL, + view = NULL, + fields = NULL, + sortField = NULL, + sortDirection = NULL, + filterByFormula = NULL, + combined_result = TRUE){ + + search_path <- table_name + + request_url <- sprintf("%s/%s/%s/listRecords", air_url, base, search_path) + request_url <- utils::URLencode(request_url) + + # append parameters to URL: + param_list <- as.list(environment())[c( + "limit", "offset", "view", "sortField", "sortDirection","filterByFormula")] + param_list <- param_list[!sapply(param_list, is.null)] + if(!is.null(fields)) { + ## allows us to use in fetch_all without having to specify a method and have + # properly formatted json + fields_unlisted <- unlist(fields) + param_list <- c(param_list, list_params(x = list(fields_unlisted), par_name = "fields")) + } + + # needed to properly format param name in json + names(param_list) <- gsub(pattern = "\\[\\d\\]",replacement = "", names(param_list) ) + + + "Note Airtable's API only accepts request with a URL shorter + than 16,000 characters. Encoded formulas may cause your + requests to exceed this limit. To fix this issue you can + instead make a POST request to + /v0/{baseId}/{tableIdOrName}/listRecords while + passing the parameters within the body of the + request instead of the query parameters." + + res <- httr::POST( + url = request_url, + config = httr::add_headers(Authorization = paste("Bearer", air_api_key())), + body = jsonlite::toJSON(param_list,auto_unbox = TRUE), + encode="raw", + httr::content_type_json() + ) + + air_validate(res) # throws exception (stop) if error + ret <- air_parse(res) # returns R object + ret_offset <- get_offset(ret) + if(combined_result) { + # combine ID, Fields and CreatedTime in the same data frame: + ret <- + cbind( + id = ret$id, ret$fields, createdTime = ret$createdTime, + stringsAsFactors =FALSE + ) + + if(!is.null(ret_offset)) { + attr(ret, "offset") <- ret_offset + } + } + + return(ret) + +} diff --git a/R/airtabler.R b/R/airtabler.R index c325c01..402f672 100644 --- a/R/airtabler.R +++ b/R/airtabler.R @@ -7,17 +7,25 @@ #' and check the API on \url{http://airtable.com/api}. #' #' @section API key: -#' Generate the airtable API key from your Airtable account page -#' (http://airtable.com/account). +#' Generate the Airtable API token from your Airtable account page +#' (http://airtable.com/create/tokens). #' #' \pkg{airtabler} functions will read the API key from #' environment variable \code{AIRTABLE_API_KEY}. To start R session with the -#' initialized environvent variable create an \code{.Renviron} file in your R home +#' initialized environment variable create an \code{.Renviron} file in your R home #' with a line like this: #' #' \code{AIRTABLE_API_KEY=************} #' #' To check where your R home is, try \code{normalizePath("~")}. +#' +#' The \pkg{usethis} and \pkg{dotenv} packages are useful for setting environment +#' variables. +#' \code{usethis::edit_r_environ} allow you to modify the \code{.Renviron} file. +#' \code{dotenv::load_dot_env} allows you to load environment variables from a +#' \code{.env} file. This second approach is especially helpful if you work +#' with multiple tokens. +#' #' @section Usage: #' Use \code{\link{airtable}} function to get airtable base object #' or just call primitives \code{\link{air_get}}, \code{\link{air_insert}}, @@ -29,7 +37,10 @@ NULL air_url <- "https://api.airtable.com/v0" +air_meta_url <- "https://api.airtable.com/v0/meta/bases" +# consider consolidating keys as the API key is now a token that can access +# the metadata api air_api_key <- function() { key <- Sys.getenv("AIRTABLE_API_KEY") if(key == "") { @@ -38,17 +49,29 @@ air_api_key <- function() { key } +# consider +air_secret_key <- function(){ + key <- Sys.getenv("AIRTABLE_SECRET_KEY") + if(key == "") { + stop("AIRTABLE_SECRET_KEY environment variable is empty. See ?airtabler for help.") + } + key +} + -#' Get a list of records or retreive a single +#' Get a list of records or retrieve a single #' -#' You can retrieve records in an order of a view by providing the name or ID of +#' Retrieve records or a single record from a table. If you provide a record_id, +#' you cannot specify fields, views, or filterFormulas. +#' +#'You can retrieve records in an order of a view by providing the name or ID of #' the view in the view query parameter. The results will include only records #' visible in the order they are displayed. #' #' @param base Airtable base #' @param table_name Table name #' @param record_id (optional) Use record ID argument to retrieve an existing -#' record details +#' record details. See \url{https://airtable.com/developers/web/api/get-record} #' @param limit (optional) A limit on the number of records to be returned. #' Limit can range between 1 and 100. #' @param offset (optional) Page offset returned by the previous list-records @@ -57,20 +80,28 @@ air_api_key <- function() { #' @param sortField (optional) The field name to use for sorting #' @param sortDirection (optional) "asc" or "desc". The sort order in which the #' records will be returned. Defaults to asc. -#' @param combined_result If TRUE (default) all data is returned in the same data -#' frame. If FALSE table fields are returned in separate \code{fields} element. +#' @param combined_result If TRUE (default) all data is returned in the same data. +#' If FALSE table fields are returned in separate \code{fields} element. +#' @param fields List. (optional) Only data for fields whose names are in this list +#' will be included in the records. Does not work when retrieving individual records with \code{record_id} +#' @param filterByFormula String. Use a formula to filter results. See \href{airtable docs}{https://support.airtable.com/hc/en-us/articles/223247187-How-to-sort-filter-or-retrieve-ordered-records-in-the-API} +#' this parameter to reduce the amount of data transferred. #' @return A data frame with records or a list with record details if #' \code{record_id} is specified. #' @export -air_get <- function(base, table_name, record_id = NULL, - limit = NULL, - offset = NULL, - view = NULL, - sortField = NULL, - sortDirection = NULL, - combined_result = TRUE) { +air_get <- function(base, table_name, + record_id = NULL, + limit = NULL, + offset = NULL, + view = NULL, + fields = NULL, + sortField = NULL, + sortDirection = NULL, + filterByFormula = NULL, + combined_result = TRUE) { search_path <- table_name + if(!missing(record_id)) { search_path <- paste0(search_path, "/", record_id) } @@ -79,25 +110,45 @@ air_get <- function(base, table_name, record_id = NULL, # append parameters to URL: param_list <- as.list(environment())[c( - "limit", "offset", "view", "sortField", "sortDirection")] + "limit", "offset", "view", "sortField", "sortDirection","filterByFormula")] param_list <- param_list[!sapply(param_list, is.null)] + if(!is.null(fields)) { + param_list <- c(param_list, list_params(x = fields, par_name = "fields")) + } + request_url <- httr::modify_url(request_url, query = param_list) - # call service: - res <- httr::GET( - url = request_url, - config = httr::add_headers(Authorization = paste("Bearer", air_api_key())) - ) - air_validate(res) # throws exception (stop) if error - ret <- air_parse(res) # returns R object - if(combined_result && is.null(record_id)) { - # combine ID, Fields and CreatedTime in the same data frame: - ret <- - cbind( - id = ret$id, ret$fields, createdTime = ret$createdTime, - stringsAsFactors =FALSE - ) + request_url <- gsub(pattern = "fields=",replacement = "fields%5B%5D=",x = request_url) + + "Note Airtable's API only accepts request with a URL shorter + than 16,000 characters. Encoded formulas may cause your + requests to exceed this limit. To fix this issue you can + instead make a POST request to + /v0/{baseId}/{tableIdOrName}/listRecords while + passing the parameters within the body of the + request instead of the query parameters." + + if(nchar(request_url) > 1600 | length(fields) > 21){ + # print("using air_post") + ret <- air_post(base,table_name,limit,offset,view,fields,sortField,sortDirection,filterByFormula,combined_result) + return(ret) + } else { + # call service: + res <- httr::GET( + url = request_url, + config = httr::add_headers(Authorization = paste("Bearer", air_api_key())) + ) + air_validate(res) # throws exception (stop) if error + ret <- air_parse(res) # returns R object + if(combined_result && is.null(record_id)) { + # combine ID, Fields and CreatedTime in the same data frame: + ret <- + cbind( + id = ret$id, ret$fields, createdTime = ret$createdTime, + stringsAsFactors =FALSE + ) + } + return(ret) } - ret } list_params <- function(x, par_name) { @@ -150,7 +201,7 @@ list_params <- function(x, par_name) { #' @param table_name Table name #' @param record_id (optional) Use record ID argument to retrieve an existing #' record details -#' @param fields (optional) Only data for fields whose names are in this vector +#' @param fields (optional) Only data for fields whose names are in this list #' will be included in the records. If you don't need every field, you can use #' this parameter to reduce the amount of data transferred. #' @param filterByFormula (optional) A formula used to filter records. @@ -170,15 +221,15 @@ list_params <- function(x, par_name) { #' \code{record_id} is specified. #' @export air_select <- function( - base, table_name, record_id = NULL, - fields = NULL, - filterByFormula = NULL, - maxRecord = NULL, - sort = NULL, - view = NULL, - pageSize = NULL, - offset = NULL, - combined_result = TRUE) { + base, table_name, record_id = NULL, + fields = NULL, + filterByFormula = NULL, + maxRecord = NULL, + sort = NULL, + view = NULL, + pageSize = NULL, + offset = NULL, + combined_result = TRUE) { search_path <- table_name if(!missing(record_id)) { @@ -255,7 +306,11 @@ air_validate <- function(res) { } air_parse <- function(res) { - res_obj <- jsonlite::fromJSON(httr::content(res, as = "text")) + if(is.character(res)){ + res_obj<- jsonlite::fromJSON(res) + } else { + res_obj <- jsonlite::fromJSON(httr::content(res, as = "text")) + } if(!is.null(res_obj$records)) { res <- res_obj$records if(!is.null(res_obj$offset)) { @@ -281,47 +336,186 @@ air_parse <- function(res) { #' @param record_data Named list of values. You can include all, some, or none #' of the field values #' @export +#' +#' air_insert <- function(base, table_name, record_data) { if( inherits(record_data, "data.frame")) { return( air_insert_data_frame(base, table_name, record_data)) } + # create json records + json_record_data <- air_make_json(base, table_name, record_data) + + # call service: + air_make_request(base,table_name,json_record_data, method = "POST") +} + + +#' Make JSON for API +#' +#' Make JSON that is compatible with the Airtable API. +#' +#' @param base String. Base in airtable +#' @param table_name String. Table in airtable +#' @param record_data Dataframe, list, or vector. Data to be converted to JSON +#' @param record_id String or vector of strings. Records to be manipulated +#' @param method String. "PATCH" is necessary for \code{air_update} +#' @param typecast Logical. Should the typecast option be TRUE or FALSE? Typecast +#' allows you to add new options to select type fields. +#' +#' @return JSON with record data +#' @export air_make_json +#' +air_make_json <- function (base, table_name, record_data, record_id = NULL, method = "POST",typecast = TRUE){ + if (inherits(record_data, "data.frame")) { + return(air_insert_data_frame(base, table_name, record_data, typecast)) + } + + #browser() record_data <- air_prepare_record(as.list(record_data)) - json_record_data <- jsonlite::toJSON(list(fields = record_data)) + fields <- list(fields = record_data) - request_url <- sprintf("%s/%s/%s", air_url, base, table_name) - # call service: - res <- httr::POST( - request_url, - httr::add_headers( - Authorization = paste("Bearer", air_api_key()), - `Content-type` = "application/json" - ), - body = json_record_data - ) + if(method == "PATCH"){ + ## the patch method already specifies a record so we + ## can remove a lot of addition json elements + + ## drop the id field as it can't be updated + fields$fields$id <- NULL - air_validate(res) # throws exception (stop) if error - air_parse(res) # returns R object + if(typecast){ + # allows us to use values not currently specified in the + # table or record eg issues = "some new issue" + # use unbox to create singleton values instead of json objects + fields$typecast <- jsonlite::unbox(typecast) + } + + + json_patch_data <- jsonlite::toJSON(fields, pretty = T) + return(json_patch_data) + } + + records <- list(records = list(fields)) + + if(typecast){ + # allows us to use values not currently specified in the + # table or record eg issues = "some new issue" + # use unbox to create singleton values instead of json objects + records$typecast <- jsonlite::unbox(typecast) + } + + json_record_data <- jsonlite::toJSON(records, pretty = T) + return(json_record_data) } -air_insert_data_frame <- function(base, table_name, records) { + +#' Make an HTTP request +#' +#' Properly encodes HTTP requests +#' +#' @param base String. Base in airtable +#' @param table_name String. Table in airtable +#' @param json_record_data json or string. JSON formatted text with record data +#' @param record_id String or vector of strings. Record id +#' @param method String. One of "POST", "PATCH", or "DELETE" +#' +#' @return Status of HTTP request +#' @export air_make_request +#' +air_make_request <- function(base, table_name, json_record_data, record_id = NULL, method = c("POST","PATCH","DELETE")){ + + if(method == "POST"){ + + request_url <- sprintf("%s/%s/%s", air_url, base, table_name) + request_url <- utils::URLencode(request_url) + + res <- httr::POST(url = request_url, + httr::add_headers( + Authorization = paste("Bearer",air_api_key()), + 'Content-type' = "application/json"), + body = json_record_data) + } + + if(method == "PATCH"){ + + ### Because the patch request url specifies the record, + ### the json does not need to be as complete + + request_url <- sprintf("%s/%s/%s/%s", air_url, base, table_name, record_id) + request_url <- utils::URLencode(request_url) + + #print(request_url) + + # browser() + + res <- httr::PATCH(url = request_url, + httr::add_headers( + Authorization = paste("Bearer",air_api_key()), + 'Content-type' = "application/json"), + body = json_record_data) + } + + if(method == "DELETE"){ + + ### Because the patch request url specifies the record, + ### the json does not need to be as complete + + request_url <- sprintf("%s/%s/%s/%s", air_url, base, table_name, record_id) + request_url <- utils::URLencode(request_url) + + # call service: + res <- httr::DELETE( + request_url, + httr::add_headers( + Authorization = paste("Bearer", air_api_key()) + ) + ) + } + + air_validate(res) # throws exception (stop) if error + air_parse(res) # returns R object +} + +#' @param base String. Airtable base +#' @param table_name String. Table name +#' @param records Dataframe. Contains records you would like to insert +#' @param typecast Logical. Should airtable make new values for select type fields? +#' +#' @rdname air_insert +#' @export air_insert_data_frame +air_insert_data_frame <- function(base, table_name, records,typecast) { lapply(seq_len(nrow(records)), function(i) { - record_data <- - as.list(records[i,]) - air_insert(base = base, table_name = table_name, record_data = record_data) + record_data <- as.list(records[i,]) + json_record_data <- air_make_json(base, table_name, record_data,typecast) + air_make_request(base = base,table_name = table_name ,json_record_data = json_record_data, method = "POST" ) + }) } +#' Update records from a dataframe +#' +#' Updates the values in a table by overwriting their current contents. +#' +#' @param base String. Airtable base +#' @param table_name String. Table name +#' @param record_ids Vector of strings. Records to be modified +#' @param records Dataframe. Values to update +#' +#' @return Status of HTTP request +#' @export air_update_data_frame +#' air_update_data_frame <- function(base, table_name, record_ids, records) { lapply(seq_len(nrow(records)), function(i) { - record_data <- - unlist(as.list(records[i,]), recursive = FALSE) - air_update(base = base, - table_name = table_name, - record_id = ifelse(is.null(record_ids), record_data$id, record_ids[i]), - record_data = record_data) + record_data <- as.list(records[i,]) + record_id <- ifelse(is.null(record_ids), record_data$id, record_ids[i]) + json_record_data <- air_make_json(base, table_name, record_data, + record_id = record_id,method = "PATCH") + + air_make_request(base = base,table_name = table_name, + json_record_data = json_record_data, + record_id = record_id, + method = "PATCH") }) } @@ -366,18 +560,11 @@ air_delete <- function(base, table_name, record_id) { return(air_delete_vec(base, table_name, record_id)) } - request_url <- sprintf("%s/%s/%s/%s", air_url, base, table_name, record_id) + air_make_request(base = base, + table_name = table_name, + record_id = record_id, + method = "DELETE") - # call service: - res <- httr::DELETE( - request_url, - httr::add_headers( - Authorization = paste("Bearer", air_api_key()) - ) - ) - - air_validate(res) # throws exception (stop) if error - air_parse(res) # returns R object } air_delete_vec <- Vectorize(air_delete, vectorize.args = "record_id", SIMPLIFY = FALSE) @@ -398,26 +585,19 @@ air_delete_vec <- Vectorize(air_delete, vectorize.args = "record_id", SIMPLIFY = #' @export air_update <- function(base, table_name, record_id, record_data) { + method <- "PATCH" + if(inherits(record_data, "data.frame")) { + #print("using data.frame method") return(air_update_data_frame(base, table_name, record_id, record_data)) } - record_data <- air_prepare_record(record_data) - json_record_data <- jsonlite::toJSON(list(fields = record_data)) - request_url <- sprintf("%s/%s/%s/%s", air_url, base, table_name, record_id) + #create json records + json_record_data <- air_make_json(base, table_name, record_data, + record_id = record_id, method = method) # call service: - res <- httr::PATCH( - request_url, - httr::add_headers( - Authorization = paste("Bearer", air_api_key()), - `Content-type` = "application/json" - ), - body = json_record_data - ) - - air_validate(res) # throws exception (stop) if error - air_parse(res) # returns R object + air_make_request(base,table_name,json_record_data,record_id = record_id, method = method) } #' Get airtable base object @@ -465,15 +645,15 @@ air_table_funs <- function(base, table_name) { res_list <- list() res_list[["select"]] <- function( - record_id = NULL, - fields = NULL, - filterByFormula = NULL, - maxRecord = NULL, - sort = NULL, - view = NULL, - pageSize = NULL, - offset = NULL, - combined_result = TRUE + record_id = NULL, + fields = NULL, + filterByFormula = NULL, + maxRecord = NULL, + sort = NULL, + view = NULL, + pageSize = NULL, + offset = NULL, + combined_result = TRUE ){ air_select(base, table_name, record_id, fields, filterByFormula, maxRecord, sort, view, @@ -506,11 +686,11 @@ air_table_funs <- function(base, table_name) { res_list[["get"]] <- function( - record_id = NULL, - limit = NULL, offset = NULL, - view = NULL, - sortField = NULL, sortDirection = NULL, - combined_result = TRUE + record_id = NULL, + limit = NULL, offset = NULL, + view = NULL, + sortField = NULL, sortDirection = NULL, + combined_result = TRUE ){ air_get(base, table_name, record_id, limit, offset, view, sortField, sortDirection, combined_result) } diff --git a/R/fetch_all.R b/R/fetch_all.R new file mode 100644 index 0000000..1f5ff88 --- /dev/null +++ b/R/fetch_all.R @@ -0,0 +1,61 @@ +#' Fetch All Records in an Airtable +#' +#' Airtable limits the number of records that can be pulled from a base to 100. +#' This function pulls records based on a query then checks if there is an +#' offset value. While there is an offset value, it uses that value to generate +#' the next query, thus moving down the records until all records have been +#' fetched from the database. +#' +#' @section Adding airtable API key to your environment: +#' Airtable requires an api key to fetch data. +#' Generate the airtable API key from your [Airtable account](http://airtable.com/account) page. +#' +#' \code{airtabler} functions will read the API key from +#' environment variable \code{AIRTABLE_API_KEY}. To start R session with the +#' initialized environment variable create an \code{.Renviron} file in your +#' home directory with a line like this: +#' +#' \code{AIRTABLE_API_KEY=your_api_key_here} +#' +#' You can use \code{usethis::edit_r_environ()} to open and edit your .Renviron +#' file. +#' +#' Also consider using the `dotenv` package with a .env file for storing +#' sensitive variables. Remember add to.gitignore or encrypt the .env file to +#' avoid sharing sensitive variables. +#' +#' @param base String. ID for the base or app to be fetched +#' @param table_name String. Name of the table to be fetched from the base +#' @param ... Additional arguments to pass to [air_get()]. \code{view} is a +#' commonly used additional argument. +#' +#' @return dataframe +#' @export fetch_all +#' +#' @examples +#' # Each base has a fully described API +#' # app_id <- "appVjIfAo8AJlfTkx" # ID for the base we are fetching. +#' # Note that you can pass a `view` argument to air_get or fetch_all to get only +#' # a view of a table (say, only validated records, or some other filtered view), +#' # e.g., +#' # bats <- fetch_all(app_id, "images", view = "Status View") +#' # talks <- fetch_all(app_id, "images") +#' +fetch_all <- function(base, table_name, ...) { + out <- list() + out[[1]] <- airtabler::air_get(base, table_name, combined_result = FALSE,...) + if(length(out[[1]]) == 0){ + emptyTableMessage <- glue::glue("The queried view for {table_name} in {base} is empty") + warning(emptyTableMessage) + return(emptyTableMessage) + } else { + offset <- airtabler::get_offset(out[[1]]) + while (!is.null(offset)) { + out <- c(out, list(airtabler::air_get(base, table_name, combined_result = FALSE, offset = offset, ...))) + offset <- airtabler::get_offset(out[[length(out)]]) + } + out <- dplyr::bind_rows(out) + cbind(id = out$id, out$fields, createdTime = out$createdTime, + stringsAsFactors = FALSE) + } +} diff --git a/R/get_id_from_url.R b/R/get_id_from_url.R new file mode 100644 index 0000000..3e51b81 --- /dev/null +++ b/R/get_id_from_url.R @@ -0,0 +1,77 @@ +#' Get an ID from a URL +#' +#' General function for parsing airtable URLs to find base, table, view, or record id's +#' +#' @param url String. A url generated by airtable +#' @param pattern String. A regex pattern for identifying the type of id you would like to get +#' @param id_type String. One of base_id, table_id, view_id, or record_id. +#' @param split_pattern String. Where should the URL be split? default is forward slashes "/" and questionmarks "?" +#' +#' @return String +#' @export air_get_id_from_url +#' +#' @examples +#' +#' url <- "https://airtable.com/apphDEokVZ9gvPNFk/tblaKC1ADBafoHVXN/viwteUgD7vaMBruHR/recMzdoM43RVRWybD?blocks=hide" +#' +#' # General function for parsing url components +#' air_get_id_from_url(url, '^app',id_type = "base_id") +#' # Get different components +#' air_get_base_id_from_url(url) +#' air_get_table_id_from_url(url) +#' air_get_view_id_from_url(url) +#' air_get_record_id_from_url(url) +#' +air_get_id_from_url <- function(url, pattern, id_type, split_pattern = "/|\\?" ){ + + url_split <- stringr::str_split_1(string = url,pattern = split_pattern) + + id_filter <- stringr::str_detect(url_split,pattern = pattern) + + if(all(!id_filter)){ + + err_msg <- glue::glue("{id_type} should conform to the pattern {pattern}.\nNo {id_type} found in {url}.") + + rlang::abort(err_msg) + } + + id <- url_split[id_filter] + + return(id) +} + + + +#' @describeIn air_get_id_from_url Get the base id +#' @export air_get_base_id_from_url +#' +air_get_base_id_from_url <- function(url,pattern = "^app\\w{13}"){ + + id <- air_get_id_from_url(url = url, pattern = pattern,id_type = "base_id") + + return(id) +} + +#' @describeIn air_get_id_from_url Get the table id +#' @export air_get_table_id_from_url +#' +air_get_table_id_from_url<- function(url, pattern = "^tbl\\w{13}"){ + id <- air_get_id_from_url(url = url, pattern = pattern,id_type = "table_id") + return(id) +} + +#' @describeIn air_get_id_from_url Get the view id +#' @export +#' +air_get_view_id_from_url<- function(url, pattern = "^viw\\w{13}"){ + id <- air_get_id_from_url(url = url, pattern = pattern,id_type = "view_id") + return(id) +} + +#' @describeIn air_get_id_from_url Get the record id +#' @export air_get_record_id_from_url +#' +air_get_record_id_from_url<- function(url, pattern = "^rec\\w{13}"){ + id <- air_get_id_from_url(url = url, pattern = pattern,id_type = "record_id") + return(id) +} diff --git a/R/get_unique_field_values.R b/R/get_unique_field_values.R new file mode 100644 index 0000000..8c82429 --- /dev/null +++ b/R/get_unique_field_values.R @@ -0,0 +1,36 @@ +#' Get unique values from a field +#' +#' Because the airtable api lacks an easy solution for getting +#' unique values from a field in a table, we have this function +#' which accepts a list of fields and returns their unique +#' values. Currently, if multiple fields are listed, all unique +#' values from all fields will be returned in a single vector. +#' This may change in future iterations. +#' +#' +#' @param base String. ID of airtable base +#' @param table_name String. Name of table in base +#' @param fields List. Names of fields +#' +#' @return vector of unique values +#' @export +#' +get_unique_field_values <- function(base, + table_name, + fields){ + + baseField <- airtabler::fetch_all(base, table_name , fields = fields ) + + if(is.character(baseField)){ + return("") + } else { + + fieldsVector <- unlist(fields) + + uniqueField <- unlist(unique(baseField[fieldsVector])) + + return(uniqueField) + + } + +} diff --git a/R/read_excel_url.R b/R/read_excel_url.R new file mode 100644 index 0000000..7ba2740 --- /dev/null +++ b/R/read_excel_url.R @@ -0,0 +1,30 @@ +#' Read an excel file from URL +#' +#' Extends \code{readxl::read_excel} to allow for reading from a URL. +#' +#' @param url String. Url for file +#' @param fileext String. File extension for temp file +#' @param ... additional arguments to pass to \code{read_excel} +#' @param parse_all_sheets Logical. Should all sheets be parsed? +#' +#' @return tibble or list of tibbles if parse_all_sheets = TRUE +#' @export read_excel_url +#' +read_excel_url <- function(url, fileext= ".xslx",parse_all_sheets = FALSE,...){ + tmp <- tempfile(fileext = ".xslx") + curl::curl_download(url, tmp ) + if(parse_all_sheets){ + sheets <- readxl::excel_sheets(tmp) + xl_list <- purrr::map(sheets,function(x){ + readxl::read_excel(path = tmp,sheet = x,...) + }) + + names(xl_list) <- sheets + + return(xl_list) + } + + readxl::read_excel(path = tmp,...) + + +} diff --git a/man/air_create_description_table.Rd b/man/air_create_description_table.Rd new file mode 100644 index 0000000..5b436ec --- /dev/null +++ b/man/air_create_description_table.Rd @@ -0,0 +1,53 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_create_description_table} +\alias{air_create_description_table} +\title{Create the descriptive metadata table for the base} +\usage{ +air_create_description_table( + base, + description, + table_name = "Description", + field_descriptions = NA, + type = "singleLineText", + options = NA +) +} +\arguments{ +\item{base}{String. Base id} + +\item{description}{Data frame. Description from air_get_base_description* or air_generate_base_description} + +\item{table_name}{String. Name of description table} + +\item{field_descriptions}{Character vector. Descriptions of metadata table fields. If NA, DCMI terms will be used where possible.} + +\item{type}{Character vector. Column types for metadata table fields. see \url{https://airtable.com/developers/web/api/field-model}} + +\item{options}{Data frame. Options for fields in metadata table.} +} +\value{ +List. Outputs from creating the table and inserting the records +} +\description{ +Descriptive metadata provides information about the base as a whole, who created it, +why, when, where can data be accessed, keywords, what license governs data use, etc. +Descriptive metadata facilitates data reuse by providing a point of contact for +future users, as well as attributes that allow the data to be entered into searchable +catalogs or archives. +} +\details{ +DCMI terms can be found here \url{https://www.dublincore.org/specifications/dublin-core/dcmi-terms/} +} +\examples{ +\dontrun{ +base = "appVjIfAo8AJlfTkx" +table_name= "description" + +description <- air_generate_base_description(title = "Example Base", + creator = "Collin Schwantes") + +air_create_description_table(base,description,table_name) +} + +} diff --git a/man/air_create_field.Rd b/man/air_create_field.Rd new file mode 100644 index 0000000..fb27ab8 --- /dev/null +++ b/man/air_create_field.Rd @@ -0,0 +1,54 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_create_field} +\alias{air_create_field} +\title{Create a new field in a table} +\usage{ +air_create_field( + base, + table_id, + name, + description = NA, + type = "singleLineText", + options = NA +) +} +\arguments{ +\item{base}{String. Base id} + +\item{table_id}{String. Table id. Can be found using \code{air_get_schema}} + +\item{name}{String. Name of the field} + +\item{description}{String. Description of the field} + +\item{type}{String. Type of field. See https://airtable.com/developers/web/api/field-model} + +\item{options}{Data frame. See https://airtable.com/developers/web/api/field-model} +} +\value{ +description of newly created field as a list +} +\description{ +See https://airtable.com/developers/web/api/create-field +} +\examples{ +\dontrun{ +base_schema <- air_get_schema(base) + +base_schema$tables + +air_create_field(base,table_id = base_schema$tables$id[[4]], + name = "Has Nucleics", + description = "Logical. Does this planet have nucleics?", + type = "checkbox", + options = list( + list( + "color"= "greenBright", + "icon"= "check" + ) + ) +) +} + +} diff --git a/man/air_create_metadata_table.Rd b/man/air_create_metadata_table.Rd new file mode 100644 index 0000000..8b8c33d --- /dev/null +++ b/man/air_create_metadata_table.Rd @@ -0,0 +1,52 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_create_metadata_table} +\alias{air_create_metadata_table} +\title{Create a new structural metadata table in the base} +\usage{ +air_create_metadata_table( + base, + meta_data, + table_name = "Meta Data", + field_descriptions = NA, + type = "singleLineText", + options = NA +) +} +\arguments{ +\item{base}{String. Base id} + +\item{meta_data}{Data frame. Contains metadata records. From air_generate_metadata*} + +\item{table_name}{String. name of the metadata table. default is "Meta Data"} + +\item{field_descriptions}{Character vector. Descriptions of metadata table fields} + +\item{type}{Character vector. Column types for metadata table fields. see https://airtable.com/developers/web/api/field-model} + +\item{options}{Data frame. Options for fields in metadata table.} +} +\value{ +List with outcome from creating the table and inserting the records +} +\description{ +Create a new structural metadata table in the base +} +\details{ +Structural metadata describes the contents of your base and how they are linked. Structural +metadata can largely be derived from the base schema. +} +\examples{ +\dontrun{ +# set base id +base <- "appXXXXXXXX" +# create metadata from api +metadata <- air_generate_metadata_from_api(base) +# add Meta Data table to base -- will not work if base already has a metadata +# table +log <- air_create_metadata_table(base,metadata) + +} + + +} diff --git a/man/air_create_table.Rd b/man/air_create_table.Rd new file mode 100644 index 0000000..9d7801c --- /dev/null +++ b/man/air_create_table.Rd @@ -0,0 +1,71 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_create_table} +\alias{air_create_table} +\title{A function to create new tables in a base} +\usage{ +air_create_table(base, table_list) +} +\arguments{ +\item{base}{String. ID for the base} + +\item{table_list}{List. see \code{air_table_template}} +} +\value{ +Data frame of table schema +} +\description{ +Takes a list object with appropriate arguments (see \code{air_table_template}) +converts it to JSON then adds it to the specified base. +} +\note{ +See https://airtable.com/developers/web/api/create-table +} +\examples{ + +\dontrun{ +base <- "appQ94sELAtFnXPxx" + +base_schema <- air_get_schema(base) + +tables<- base_schema$tables + +field_names <- c("Planet","Chapter","Book", "Known Inhabitants") + +field_desc <- c("Name of planet in Foundation Series", + "Chapters where planet is referenced", + "Books where planet is referenced", + "Characters mentioned as living on or being from that planet") + +field_types <- c("singleLineText",rep("multipleRecordLinks",3)) + +field_options <- c(NA,list( + list( + linkedTableId = tables[tables$name == "Chapter","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Book","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Character","id"] + ) +) +) + +field_df<- air_fields_df_template(name = field_names, + description = field_desc, + type = field_types, + options = field_options) + +table_list <- air_table_template(table_name = "Planet", + description = "Planets of Foundation", + fields_df = field_df) + +air_create_table(base, table_list) + +} +} diff --git a/man/air_download_attachments.Rd b/man/air_download_attachments.Rd new file mode 100644 index 0000000..793bdbd --- /dev/null +++ b/man/air_download_attachments.Rd @@ -0,0 +1,53 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_download_attachments.R +\name{air_download_attachments} +\alias{air_download_attachments} +\title{Download Airtable file attachments} +\usage{ +air_download_attachments( + x, + field, + dir_name = "downloads", + include_attachment_id = TRUE, + ... +) +} +\arguments{ +\item{x}{Data frame. Output from air_get or fetch_all.} + +\item{field}{String. Name of field with file attachments in base} + +\item{dir_name}{String. Where should files be downloaded to? +Will create the folder if it does not exist. Folders created are recursively.} + +\item{include_attachment_id}{Logical. Should you include the airtable attachment +ID to guarantee all file names are unique? Default is true.} + +\item{...}{reserved for additional arguments.} +} +\value{ +Returns x with an additional field called attachment_file_paths +} +\description{ +Download an attachment stored in air tables. Returns original dataframe +with an additional field called attachment_file_paths. The attachment_file_paths +field is of class list so it can handle multiple attachments per record. File +paths are prepended with record ids so that all file names are unique. +} +\examples{ +\dontrun{ + +base <- "appXXXXXXXXX" +table_name <- "Table With Attachments" + +table_original <- air_get(base,table_name) + +table_with_file_paths <- air_download_attachments(x = table_with_attachments, + field = "attachment_field", + dir_name = "downloads") + +table_with_file_paths$attachment_file_paths + +} + +} diff --git a/man/air_dump.Rd b/man/air_dump.Rd new file mode 100644 index 0000000..e0ed61e --- /dev/null +++ b/man/air_dump.Rd @@ -0,0 +1,55 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_dump} +\alias{air_dump} +\title{Dump all tables from a base into R} +\usage{ +air_dump( + base, + metadata = NULL, + description = NULL, + add_missing_fields = TRUE, + download_attachments = TRUE, + attachment_fields = NULL, + polite_downloads = TRUE, + field_names_to_snakecase = TRUE, + ... +) +} +\arguments{ +\item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} + +\item{metadata}{Data.frame.Data frame with structural metadata - describes relationship between tables and fields. +Can be left as NULL if base already contains a table called meta data.} + +\item{description}{Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. +Can be left as NULL if base already contains a table called description.} + +\item{add_missing_fields}{Logical. Should fields described in the metadata data.frame be added to corresponding tables?} + +\item{download_attachments}{Logical. Should attached files be downloaded?} + +\item{attachment_fields}{Optional. character vector. +What field(s) should files be downloaded from? Default is to download all fields +with type multipleAttachments in metadata.} + +\item{polite_downloads}{Logical. Use if downloading many files. Sets a delay +so that server is not overwhelmed by requests.} + +\item{field_names_to_snakecase}{Logical. Should field names be +converted to snake case?} + +\item{...}{Additional arguments to pass to air_download_attachments} +} +\value{ +List of data.frames. All tables from metadata plus the +description and metadata tables. +} +\description{ +Dump all tables from a base into R +} +\note{ +To facilitate joining on ids, see purrr::as_vector for converting list + type columns to vectors and +tidyr::unnest for expanding list columns. +} diff --git a/man/air_dump_to_csv.Rd b/man/air_dump_to_csv.Rd new file mode 100644 index 0000000..4120fbd --- /dev/null +++ b/man/air_dump_to_csv.Rd @@ -0,0 +1,38 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_dump_to_csv} +\alias{air_dump_to_csv} +\title{Save air_dump output to csv} +\usage{ +air_dump_to_csv( + table_list, + output_dir = "outputs", + attachments_dir = NULL, + overwrite = FALSE, + output_id = NULL, + names_to_snake_case = TRUE +) +} +\arguments{ +\item{table_list}{List. List of data.frames output from \code{air_dump}} + +\item{output_dir}{String. Folder containing output files} + +\item{attachments_dir}{String. What folder are base attachments stored in?} + +\item{overwrite}{Logical. Should outputs be overwritten if they already exist?} + +\item{output_id}{String. Optional identifier for the data set - if NULL an +ID will be generated using a hash of the data.} + +\item{names_to_snake_case}{Logical. Should field and table names be converted to snake_case?} +} +\value{ +Vector of file paths +} +\description{ +Saves data.frames from air_dump to csv files. File names are determined by +the names of the list objects from air_dump. Files will be saved in folder +with a unique name, inside the folder specified by \code{output_dir}. The +unique name is generated from a hash of the air_dump output. +} diff --git a/man/air_dump_to_json.Rd b/man/air_dump_to_json.Rd new file mode 100644 index 0000000..a7a4de7 --- /dev/null +++ b/man/air_dump_to_json.Rd @@ -0,0 +1,34 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_dump_to_json} +\alias{air_dump_to_json} +\title{Dump all tables from a base into json files} +\usage{ +air_dump_to_json( + base, + metadata, + description = NULL, + output_dir = "outputs", + overwrite = FALSE +) +} +\arguments{ +\item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} + +\item{metadata}{Data.frame.Data frame with structural metadata - describes relationship between tables and fields.} + +\item{description}{Data.frame. Data frame with descriptive metadata - describes whats in your base and who made it. +Can be left as NULL if base already contains a table called description} + +\item{output_dir}{String. Where should json files be saved?} + +\item{overwrite}{Logical. If data are not unique, should files be overwritten?} +} +\value{ +List of data.frames. All tables from metadata plus the +description and metadata tables. +} +\description{ +Essentially air_get without converting to Rs. Does not add fields with empty +values. +} diff --git a/man/air_expand_csv_arrays.Rd b/man/air_expand_csv_arrays.Rd new file mode 100644 index 0000000..5ab6746 --- /dev/null +++ b/man/air_expand_csv_arrays.Rd @@ -0,0 +1,42 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_expand_csv_arrays.R +\name{air_expand_csv_arrays} +\alias{air_expand_csv_arrays} +\title{Expand arrays stored in CSVs} +\usage{ +air_expand_csv_arrays(x, simplify_to_vector = FALSE) +} +\arguments{ +\item{x}{Character. likely a vector or field in a dataframe.} + +\item{simplify_to_vector}{Logical. Should expanded arrays be converted from +lists to vectors? For lists with multiple elements at a given position, the length of the output +may be greater than the length of the input. See [tidyr::unnest()] for expanding +list columns.} +} +\value{ +A vector or list of expanded arrays. +} +\description{ +This function helps users work with airtable data that has been exported to CSVs. +Because airtable uses nested data structures (json arrays), the data must be +flattened to be stored in a csv. The standard way to store arrays in a csv +is to wrap the array in quotes and separate each item with commas. So an array +stored in a csv would look like "item 1,item 2,...,item n". This function will +convert arrays stored in csvs to either a list or a vector and removes the +surrounding quotes. +} +\examples{ + + +# example vector data +x <- c("item 1,item 2,item 3","apple,orange,banana","1,2,3","") + +# to list +air_expand_csv_arrays(x) + +# to vector +air_expand_csv_arrays(x,simplify_to_vector = TRUE) + + +} diff --git a/man/air_fields_df_template.Rd b/man/air_fields_df_template.Rd new file mode 100644 index 0000000..e471c7d --- /dev/null +++ b/man/air_fields_df_template.Rd @@ -0,0 +1,71 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_fields_df_template} +\alias{air_fields_df_template} +\title{Template for for creating a table from a tibble} +\usage{ +air_fields_df_template(name, description, type, options = NA) +} +\arguments{ +\item{name}{String. Names of fields in the table} + +\item{description}{String. Descriptions of fields} + +\item{type}{String. Type of columns. For values see \url{https://airtable.com/developers/web/api/model/field-type}} + +\item{options}{List. Options will be converted from lists to JSON. For field options see \url{https://airtable.com/developers/web/api/field-model}} +} +\value{ +Tibble with attributes required for fields in a table +} +\description{ +Convenience function for creating the content of tables that will created or +updated viaAPI. +} +\examples{ + +\dontrun{ +base <- "appQ94sELAtFnXPxx" + +base_schema <- air_get_schema(base) + +tables<- base_schema$tables + +field_names <- c("Planet","Chapter","Book", "Known Inhabitants") + +field_desc <- c("Name of planet in Foundation Series", + "Chapters where planet is referenced", + "Books where planet is referenced", + "Characters mentioned as living on or being from that planet") + +field_types <- c("singleLineText",rep("multipleRecordLinks",3)) + +field_options <- c(NA,list( + list( + linkedTableId = tables[tables$name == "Chapter","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Book","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Character","id"] + ) +) +) + +field_df<- air_fields_df_template(name = field_names, + description = field_desc, + type = field_types, + options = field_options) + +table_list <- air_table_template(table_name = "Planet", + description = "Planets of Foundation", + fields_df = field_tables) + +air_create_table(base, table_list) +} +} diff --git a/man/air_fields_list_from_template.Rd b/man/air_fields_list_from_template.Rd new file mode 100644 index 0000000..d7fac4d --- /dev/null +++ b/man/air_fields_list_from_template.Rd @@ -0,0 +1,60 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_fields_list_from_template} +\alias{air_fields_list_from_template} +\title{Convert field data frame to list} +\usage{ +air_fields_list_from_template(df) +} +\arguments{ +\item{df}{Data frame. From air_fields_df_template} +} +\value{ +List. Structured for easy parsing into JSON +} +\description{ +Converts the field data frame to a list of easier translation to JSON +} +\examples{ +\dontrun{ +base <- "appQ94sELAtFnXPxx" + +base_schema <- air_get_schema(base) + +tables<- base_schema$tables + +field_names <- c("Planet","Chapter","Book", "Known Inhabitants") + +field_desc <- c("Name of planet in Foundation Series", + "Chapters where planet is referenced", + "Books where planet is referenced", + "Characters mentioned as living on or being from that planet") + +field_types <- c("singleLineText",rep("multipleRecordLinks",3)) + +field_options <- c(NA,list( + list( + linkedTableId = tables[tables$name == "Chapter","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Book","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Character","id"] + ) +) +) + +field_df <- air_fields_df_template(name = field_names, + description = field_desc, + type = field_types, + options = field_options) + +fields_list <- air_fields_list_from_template(df = fields_df) + +} +} diff --git a/man/air_generate_base_description.Rd b/man/air_generate_base_description.Rd new file mode 100644 index 0000000..8686581 --- /dev/null +++ b/man/air_generate_base_description.Rd @@ -0,0 +1,74 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_generate_base_description} +\alias{air_generate_base_description} +\title{Generate descriptive metadata} +\usage{ +air_generate_base_description( + title = NA, + creator = NA, + created = NA, + primary_contact = NA, + email = NA, + description = NA, + contributor = NA, + identifier = NA, + license = NA, + ... +) +} +\arguments{ +\item{title}{String. Title is a property that refers to the name or names by +which a resource is formally known.} + +\item{creator}{String. Person or people who created the base} + +\item{created}{String. When was the base created?} + +\item{primary_contact}{String. Person or entity primarily responsible for +making the content of a resource} + +\item{email}{String. Email of primary_contact} + +\item{description}{String. This property refers to the description of +the content of a resource. The description is a potentially rich source of +indexable terms and assist the users in their selection of an appropriate +resource.} + +\item{contributor}{String. An entity responsible for making contributions to the resource.} + +\item{identifier}{String. An unambiguous reference to the resource within a given context.} + +\item{license}{String. A legal document giving official permission to do something with the resource. "CC BY 4.0"} + +\item{...}{String. Additional descriptive metadata elements. See details. +Additional elements can be added as name pair values e.g. +\code{ isPartOf = "https://doi.org/00.00000/MyPaper01", isReferencedBy = "https://doi.org/10.48321/MyDMP01"}} +} +\value{ +data.frame with descriptive metadata +} +\description{ +Creates a data.frame that describes the base. Descriptive metadata provides +information about the base as a whole: who created it, +why, when, where can data be accessed, keywords, what license governs data use, etc. +Descriptive metadata facilitates data reuse by providing a point of contact for +future users, as well as attributes that allow the data to be entered into searchable +catalogs or archives. +} +\details{ +See \href{https://www.dublincore.org/resources/userguide/creating_metadata/}{dublin core} for inspiration about additional attributes. +} +\examples{ + +air_generate_base_description(title = "My Awesome Base" , + primary_contact= "Base Creator/Maintainer", + email = "email@example.com", + base_description = "This base contains my awesome data + from a project studying XXX in YYY. Data in the base were collected + from 1900-01-01 to 1990-01-01 by researchers at Some Long Term Project.", + is_part_of = "https://doi.org/10.48321/MyDMP01", + isReferencedBy = "https://doi.org/10.5072/zenodo_sandbox.1062705" + ) + +} diff --git a/man/air_generate_metadata_from_api.Rd b/man/air_generate_metadata_from_api.Rd new file mode 100644 index 0000000..bccfda2 --- /dev/null +++ b/man/air_generate_metadata_from_api.Rd @@ -0,0 +1,45 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_generate_metadata_from_api} +\alias{air_generate_metadata_from_api} +\title{Generate structural metadata from the api} +\usage{ +air_generate_metadata_from_api( + base, + metadata_table_name = "Meta Data", + include_metadata_table = FALSE, + field_names_to_snake_case = TRUE +) +} +\arguments{ +\item{base}{String. Base id} + +\item{metadata_table_name}{String. Name of exisiting structural metadata table if it exists} + +\item{include_metadata_table}{Logical. Should the structural metadata table be included in the metadata?} + +\item{field_names_to_snake_case}{Logical. Should the field names in the metadata table be snake_case?} +} +\value{ +A data frame with metadata +} +\description{ +Structural metadata describes the contents of your base and how they are linked. +The structural metadata are created from the base schema. The nested schema +structure is flattened into a more user-friendly table which can then be +inserted as a table into the base with \code{air_created_metadata_table} and/or +used in a data export with \code{air_dump}. +} +\details{ +This function requires that the api token has the ability to read +the base schema. +} +\examples{ + +\dontrun{ + +base <- "appXXXXXXXX" +metadata <- air_generate_metadata_from_api(base) + +} +} diff --git a/man/air_generate_metadata_from_tables.Rd b/man/air_generate_metadata_from_tables.Rd new file mode 100644 index 0000000..28576f1 --- /dev/null +++ b/man/air_generate_metadata_from_tables.Rd @@ -0,0 +1,31 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_generate_metadata_from_tables} +\alias{air_generate_metadata_from_tables} +\title{Generated Metadata from table names} +\usage{ +air_generate_metadata_from_tables(base, table_names, limit = 1) +} +\arguments{ +\item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} + +\item{table_names}{Vector of strings. The names of your tables. eg c("table 1","table 2", etc.)} + +\item{limit}{Number from 1-100. How many rows should we pull from each table to create the metdata? +Keep in mind that the airtable api will not return fields with "empty" values - "", false, or []. +Code runs faster if fewer rows are pulled.} +} +\value{ +data.frame with structural metadata. +} +\description{ +Deprecated: Use \code{air_generate_metadata_from_api} +} +\details{ +Generates a structural metadata table - the metadata that +describes how tables and fields fit together. Does not +include field types. + +For information about creating metadata tables in your base see the +\href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} +} diff --git a/man/air_get.Rd b/man/air_get.Rd index 0f9d3f5..97bfe47 100644 --- a/man/air_get.Rd +++ b/man/air_get.Rd @@ -2,11 +2,21 @@ % Please edit documentation in R/airtabler.R \name{air_get} \alias{air_get} -\title{Get a list of records or retreive a single} +\title{Get a list of records or retrieve a single} \usage{ -air_get(base, table_name, record_id = NULL, limit = NULL, offset = NULL, - view = NULL, sortField = NULL, sortDirection = NULL, - combined_result = TRUE) +air_get( + base, + table_name, + record_id = NULL, + limit = NULL, + offset = NULL, + view = NULL, + fields = NULL, + sortField = NULL, + sortDirection = NULL, + filterByFormula = NULL, + combined_result = TRUE +) } \arguments{ \item{base}{Airtable base} @@ -14,7 +24,7 @@ air_get(base, table_name, record_id = NULL, limit = NULL, offset = NULL, \item{table_name}{Table name} \item{record_id}{(optional) Use record ID argument to retrieve an existing -record details} +record details. See \url{https://airtable.com/developers/web/api/get-record}} \item{limit}{(optional) A limit on the number of records to be returned. Limit can range between 1 and 100.} @@ -24,19 +34,29 @@ call. Note that this is represented by a record ID, not a numerical offset.} \item{view}{(optional) The name or ID of the view} +\item{fields}{List. (optional) Only data for fields whose names are in this list +will be included in the records. Does not work when retrieving individual records with \code{record_id}} + \item{sortField}{(optional) The field name to use for sorting} \item{sortDirection}{(optional) "asc" or "desc". The sort order in which the records will be returned. Defaults to asc.} -\item{combined_result}{If TRUE (default) all data is returned in the same data -frame. If FALSE table fields are returned in separate \code{fields} element.} +\item{filterByFormula}{String. Use a formula to filter results. See \href{airtable docs}{https://support.airtable.com/hc/en-us/articles/223247187-How-to-sort-filter-or-retrieve-ordered-records-in-the-API} +this parameter to reduce the amount of data transferred.} + +\item{combined_result}{If TRUE (default) all data is returned in the same data. +If FALSE table fields are returned in separate \code{fields} element.} } \value{ A data frame with records or a list with record details if \code{record_id} is specified. } \description{ +Retrieve records or a single record from a table. If you provide a record_id, +you cannot specify fields, views, or filterFormulas. +} +\details{ You can retrieve records in an order of a view by providing the name or ID of the view in the view query parameter. The results will include only records visible in the order they are displayed. diff --git a/man/air_get_attachments.Rd b/man/air_get_attachments.Rd new file mode 100644 index 0000000..efad70f --- /dev/null +++ b/man/air_get_attachments.Rd @@ -0,0 +1,70 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_get_attachments.R +\name{air_get_attachments} +\alias{air_get_attachments} +\title{Get Airtable file attachments} +\usage{ +air_get_attachments( + base, + table_name, + field, + download_file = FALSE, + include_attachment_id = TRUE, + dir_name = "downloads", + extract_type = "excel", + extract_field = "excel_extract", + skip = 0, + parse_all_sheets = FALSE, + ... +) +} +\arguments{ +\item{base}{String. ID for the base or app to be fetched} + +\item{table_name}{String. Name of the table to be fetched from the base} + +\item{field}{String. Name of field with file attachments in base} + +\item{download_file}{Logical. Should files be downloaded?} + +\item{include_attachment_id}{Logical. Should the attachment ID be included in the file name? +Default is true to ensure unique file names.} + +\item{dir_name}{String. Where should files be downloaded to? +Will create the folder if it does not exist.} + +\item{extract_type}{String. File type to be extracted. +Should be one of: excel} + +\item{extract_field}{String. Name of extract field that will be created} + +\item{skip}{Numeric. How many lines should be skipped? See \code{readxl::read_excel} skip.} + +\item{parse_all_sheets}{Logical. Should all sheets in spreadsheet be parsed?} + +\item{...}{Additional arguments to pass to \code{air_get}} +} +\value{ +named list of data frames +} +\description{ +Extract the contents of an attachment stored in Airtable. Currently only setup +to work with Excel files. Planned expansion to other file types. +For excel files, returns a named list. +} +\examples{ + +\dontrun{ + +base <- "appXXXXXXXXX" +table_name <- "table with excel attachments" + + table_with_attachments <- air_get_attachments(base,table_name, field = "attachment_field" ) + +} + + +} +\seealso{ +\code{air_download_attachments} +} diff --git a/man/air_get_base_description_from_table.Rd b/man/air_get_base_description_from_table.Rd new file mode 100644 index 0000000..bb26349 --- /dev/null +++ b/man/air_get_base_description_from_table.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_get_base_description_from_table} +\alias{air_get_base_description_from_table} +\title{Get base description from table} +\usage{ +air_get_base_description_from_table( + base, + table_name, + field_names_to_snakecase = TRUE +) +} +\arguments{ +\item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} + +\item{table_name}{String. Name of descriptive metadata table - the metadata that +describes the base and provides attribution} + +\item{field_names_to_snakecase}{Logical. Should field names be converted to snakecase?} +} +\value{ +data.frame with descriptive metadata. +} +\description{ +Pull a table that has descriptive metadata. +Requires the following fields: +"title","primary_contact","email","description" +} +\examples{ +\dontrun{ +base <- "appXXXXXXXX" +table_name <- "Description" +air_get_base_description_from_table(base, table_name) +} +} diff --git a/man/air_get_id_from_url.Rd b/man/air_get_id_from_url.Rd new file mode 100644 index 0000000..4304581 --- /dev/null +++ b/man/air_get_id_from_url.Rd @@ -0,0 +1,59 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_id_from_url.R +\name{air_get_id_from_url} +\alias{air_get_id_from_url} +\alias{air_get_base_id_from_url} +\alias{air_get_table_id_from_url} +\alias{air_get_view_id_from_url} +\alias{air_get_record_id_from_url} +\title{Get an ID from a URL} +\usage{ +air_get_id_from_url(url, pattern, id_type, split_pattern = "/|\\\\?") + +air_get_base_id_from_url(url, pattern = "^app\\\\w{13}") + +air_get_table_id_from_url(url, pattern = "^tbl\\\\w{13}") + +air_get_view_id_from_url(url, pattern = "^viw\\\\w{13}") + +air_get_record_id_from_url(url, pattern = "^rec\\\\w{13}") +} +\arguments{ +\item{url}{String. A url generated by airtable} + +\item{pattern}{String. A regex pattern for identifying the type of id you would like to get} + +\item{id_type}{String. One of base_id, table_id, view_id, or record_id.} + +\item{split_pattern}{String. Where should the URL be split? default is forward slashes "/" and questionmarks "?"} +} +\value{ +String +} +\description{ +General function for parsing airtable URLs to find base, table, view, or record id's +} +\section{Functions}{ +\itemize{ +\item \code{air_get_base_id_from_url()}: Get the base id + +\item \code{air_get_table_id_from_url()}: Get the table id + +\item \code{air_get_view_id_from_url()}: Get the view id + +\item \code{air_get_record_id_from_url()}: Get the record id + +}} +\examples{ + +url <- "https://airtable.com/apphDEokVZ9gvPNFk/tblaKC1ADBafoHVXN/viwteUgD7vaMBruHR/recMzdoM43RVRWybD?blocks=hide" + +# General function for parsing url components +air_get_id_from_url(url, '^app',id_type = "base_id") +# Get different components +air_get_base_id_from_url(url) +air_get_table_id_from_url(url) +air_get_view_id_from_url(url) +air_get_record_id_from_url(url) + +} diff --git a/man/air_get_json.Rd b/man/air_get_json.Rd new file mode 100644 index 0000000..7bebb86 --- /dev/null +++ b/man/air_get_json.Rd @@ -0,0 +1,57 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_get_json.R +\name{air_get_json} +\alias{air_get_json} +\title{Get a list of records or retrieve a single record as JSON} +\usage{ +air_get_json( + base, + table_name, + record_id = NULL, + limit = NULL, + offset = NULL, + view = NULL, + fields = NULL, + sortField = NULL, + sortDirection = NULL, + combined_result = TRUE, + pretty = FALSE +) +} +\arguments{ +\item{base}{Airtable base} + +\item{table_name}{Table name} + +\item{record_id}{(optional) Use record ID argument to retrieve an existing +record details} + +\item{limit}{(optional) A limit on the number of records to be returned. +Limit can range between 1 and 100.} + +\item{offset}{(optional) Page offset returned by the previous list-records +call. Note that this is represented by a record ID, not a numerical offset.} + +\item{view}{(optional) The name or ID of the view} + +\item{fields}{(optional) Only data for fields whose names are in this list +will be included in the records. If you don't need every field, you can use} + +\item{sortField}{(optional) The field name to use for sorting} + +\item{sortDirection}{(optional) "asc" or "desc". The sort order in which the +records will be returned. Defaults to asc.} + +\item{combined_result}{If TRUE (default) all data is returned in the same data. +If FALSE table fields are returned in separate \code{fields} element.} + +\item{pretty}{Logical. Should JSON be returned in human readable form? +this parameter to reduce the amount of data transferred.} +} +\value{ +A data frame with records or a list with record details if + \code{record_id} is specified. +} +\description{ +Returns JSON objects from GET requests +} diff --git a/man/air_get_metadata_from_table.Rd b/man/air_get_metadata_from_table.Rd new file mode 100644 index 0000000..10498d4 --- /dev/null +++ b/man/air_get_metadata_from_table.Rd @@ -0,0 +1,40 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_get_metadata_from_table} +\alias{air_get_metadata_from_table} +\title{Pull the metadata table from Airtable} +\usage{ +air_get_metadata_from_table( + base, + table_name, + add_id_field = FALSE, + field_names_to_snakecase = TRUE +) +} +\arguments{ +\item{base}{String. ID for your base from Airtable. Generally 'appXXXXXXXXXXXXXX'} + +\item{table_name}{String. Name of structural metadata table - the metadata that +describes how tables and fields fit together.} + +\item{add_id_field}{Logical. If true, an "id" field is added to each table} + +\item{field_names_to_snakecase}{Logical. If true, values in the field_names +column and the field in the metadata table themselves are are converted to snake_case} +} +\value{ +data.frame with metadata table +} +\description{ +Airtable allows all users to access the metadata API. +The recommended workflow for creating this table is to use +air_generate_metadata_from_api to extract the structural metadata from the base +schema and then use air_create_metadata_table to add the table to your +base. +} +\details{ +For information about creating metadata tables in your base see the +\href{https://ecohealthalliance.github.io/eha-ma-handbook/8-airtable.html#managing-data}{EHA MA Handbook} + +Requires the following fields: table_name, field_name +} diff --git a/man/air_get_schema.Rd b/man/air_get_schema.Rd new file mode 100644 index 0000000..c11686e --- /dev/null +++ b/man/air_get_schema.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_get_schema} +\alias{air_get_schema} +\title{Get base schema} +\usage{ +air_get_schema(base, ...) +} +\arguments{ +\item{base}{String. Airtable base ID} + +\item{...}{reserved for additional parameters} +} +\value{ +list of schema +} +\description{ +Get the schema for the tables in a base. This is a wrapper for the api call +Get base schema. +} +\section{Using Metadata API}{ + +Metadata api is currently available to all users. +} + diff --git a/man/air_insert.Rd b/man/air_insert.Rd index 6e22eba..9c605c8 100644 --- a/man/air_insert.Rd +++ b/man/air_insert.Rd @@ -2,21 +2,28 @@ % Please edit documentation in R/airtabler.R \name{air_insert} \alias{air_insert} +\alias{air_insert_data_frame} \alias{multiple} \title{Insert a new record} \usage{ air_insert(base, table_name, record_data) +air_insert_data_frame(base, table_name, records, typecast) + multiple(x) } \arguments{ -\item{base}{Airtable base} +\item{base}{String. Airtable base} -\item{table_name}{Table name} +\item{table_name}{String. Table name} \item{record_data}{Named list of values. You can include all, some, or none of the field values} +\item{records}{Dataframe. Contains records you would like to insert} + +\item{typecast}{Logical. Should airtable make new values for select type fields?} + \item{x}{Object to be marked as a multiple value field} } \description{ diff --git a/man/air_list_bases.Rd b/man/air_list_bases.Rd new file mode 100644 index 0000000..842698e --- /dev/null +++ b/man/air_list_bases.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_list_bases} +\alias{air_list_bases} +\title{Get list of bases for an Token} +\usage{ +air_list_bases(request_url = "https://api.airtable.com/v0/meta/bases") +} +\arguments{ +\item{request_url}{String. URL for api endpoint} +} +\value{ +list. List of bases a token can access. +} +\description{ +Each token you provisision is given access to a certain set of bases or +workspaces. This function lists all bases associated with a token. +} +\examples{ + +\dontrun{ +air_list_bases() +} + +} diff --git a/man/air_make_json.Rd b/man/air_make_json.Rd new file mode 100644 index 0000000..541c47a --- /dev/null +++ b/man/air_make_json.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/airtabler.R +\name{air_make_json} +\alias{air_make_json} +\title{Make JSON for API} +\usage{ +air_make_json( + base, + table_name, + record_data, + record_id = NULL, + method = "POST", + typecast = TRUE +) +} +\arguments{ +\item{base}{String. Base in airtable} + +\item{table_name}{String. Table in airtable} + +\item{record_data}{Dataframe, list, or vector. Data to be converted to JSON} + +\item{record_id}{String or vector of strings. Records to be manipulated} + +\item{method}{String. "PATCH" is necessary for \code{air_update}} + +\item{typecast}{Logical. Should the typecast option be TRUE or FALSE? Typecast +allows you to add new options to select type fields.} +} +\value{ +JSON with record data +} +\description{ +Make JSON that is compatible with the Airtable API. +} diff --git a/man/air_make_request.Rd b/man/air_make_request.Rd new file mode 100644 index 0000000..d5f1257 --- /dev/null +++ b/man/air_make_request.Rd @@ -0,0 +1,31 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/airtabler.R +\name{air_make_request} +\alias{air_make_request} +\title{Make an HTTP request} +\usage{ +air_make_request( + base, + table_name, + json_record_data, + record_id = NULL, + method = c("POST", "PATCH", "DELETE") +) +} +\arguments{ +\item{base}{String. Base in airtable} + +\item{table_name}{String. Table in airtable} + +\item{json_record_data}{json or string. JSON formatted text with record data} + +\item{record_id}{String or vector of strings. Record id} + +\item{method}{String. One of "POST", "PATCH", or "DELETE"} +} +\value{ +Status of HTTP request +} +\description{ +Properly encodes HTTP requests +} diff --git a/man/air_post.Rd b/man/air_post.Rd new file mode 100644 index 0000000..afa1116 --- /dev/null +++ b/man/air_post.Rd @@ -0,0 +1,61 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_post.R +\name{air_post} +\alias{air_post} +\title{Get a list of records} +\usage{ +air_post( + base, + table_name, + limit = NULL, + offset = NULL, + view = NULL, + fields = NULL, + sortField = NULL, + sortDirection = NULL, + filterByFormula = NULL, + combined_result = TRUE +) +} +\arguments{ +\item{base}{String. Airtable base} + +\item{table_name}{String. Table name} + +\item{limit}{Numeric. (optional) A limit on the number of records to be returned. +Limit can range between 1 and 100.} + +\item{offset}{Numeric. (optional) Page offset returned by the previous list-records +call. Note that this is represented by a record ID, not a numerical offset.} + +\item{view}{String. (optional) The name or ID of the view} + +\item{fields}{List. (optional) Only data for fields whose names are in this list +will be included in the records. Does not work when retrieving individual records with \code{record_id}} + +\item{sortField}{String. (optional) The field name to use for sorting} + +\item{sortDirection}{String. (optional) "asc" or "desc". The sort order in which the +records will be returned. Defaults to asc.} + +\item{filterByFormula}{String. Use a formula to filter results. See \href{airtable docs}{https://support.airtable.com/hc/en-us/articles/223247187-How-to-sort-filter-or-retrieve-ordered-records-in-the-API} +this parameter to reduce the amount of data transferred.} + +\item{combined_result}{Logical. If TRUE (default) all data is returned in the same data. +If FALSE table fields are returned in separate \code{fields} element.} +} +\value{ +A data frame with records +} +\description{ +Retrieve records where the request url would be over 16k characters (e.g. +complicated formula) or has more than 21 fields listed in the request. +} +\details{ +You can retrieve records in an order of a view by providing the name or ID of +the view in the view query parameter. The results will include only records +visible in the order they are displayed. +} +\seealso{ +[air_get()] +} diff --git a/man/air_select.Rd b/man/air_select.Rd index 00fa4ef..5b246c2 100644 --- a/man/air_select.Rd +++ b/man/air_select.Rd @@ -4,9 +4,19 @@ \alias{air_select} \title{Select} \usage{ -air_select(base, table_name, record_id = NULL, fields = NULL, - filterByFormula = NULL, maxRecord = NULL, sort = NULL, view = NULL, - pageSize = NULL, offset = NULL, combined_result = TRUE) +air_select( + base, + table_name, + record_id = NULL, + fields = NULL, + filterByFormula = NULL, + maxRecord = NULL, + sort = NULL, + view = NULL, + pageSize = NULL, + offset = NULL, + combined_result = TRUE +) } \arguments{ \item{base}{Airtable base} @@ -16,7 +26,7 @@ air_select(base, table_name, record_id = NULL, fields = NULL, \item{record_id}{(optional) Use record ID argument to retrieve an existing record details} -\item{fields}{(optional) Only data for fields whose names are in this vector +\item{fields}{(optional) Only data for fields whose names are in this list will be included in the records. If you don't need every field, you can use this parameter to reduce the amount of data transferred.} diff --git a/man/air_table_template.Rd b/man/air_table_template.Rd new file mode 100644 index 0000000..b5e69df --- /dev/null +++ b/man/air_table_template.Rd @@ -0,0 +1,72 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_table_template} +\alias{air_table_template} +\title{Template for lists that describe tables in Airtable} +\usage{ +air_table_template(table_name, description, fields_df) +} +\arguments{ +\item{table_name}{String. Name of table} + +\item{description}{String. Description of the table} + +\item{fields_df}{Data frame. Data frame describing the field in a table. +Should contain a name, description,type, and options field. +if} +} +\value{ +List with table name, description, and fields +} +\description{ +Template for lists that describe tables in Airtable +} +\examples{ + +\dontrun{ +base <- "appQ94sELAtFnXPxx" + +base_schema <- air_get_schema(base) + +tables<- base_schema$tables + +field_names <- c("Planet","Chapter","Book", "Known Inhabitants") + +field_desc <- c("Name of planet in Foundation Series", + "Chapters where planet is referenced", + "Books where planet is referenced", + "Characters mentioned as living on or being from that planet") + +field_types <- c("singleLineText",rep("multipleRecordLinks",3)) + +field_options <- c(NA,list( + list( + linkedTableId = tables[tables$name == "Chapter","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Book","id"] + ) +), +list( + list( + linkedTableId = tables[tables$name == "Character","id"] + ) +) +) + +field_df<- air_fields_df_template(name = field_names, + description = field_desc, + type = field_types, + options = field_options) + +table_list <- air_table_template(table_name = "Planet", + description = "Planets of Foundation", + fields_df = field_tables) + +air_create_table(base, table_list) +} + + +} diff --git a/man/air_update_data_frame.Rd b/man/air_update_data_frame.Rd new file mode 100644 index 0000000..e736380 --- /dev/null +++ b/man/air_update_data_frame.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/airtabler.R +\name{air_update_data_frame} +\alias{air_update_data_frame} +\title{Update records from a dataframe} +\usage{ +air_update_data_frame(base, table_name, record_ids, records) +} +\arguments{ +\item{base}{String. Airtable base} + +\item{table_name}{String. Table name} + +\item{record_ids}{Vector of strings. Records to be modified} + +\item{records}{Dataframe. Values to update} +} +\value{ +Status of HTTP request +} +\description{ +Updates the values in a table by overwriting their current contents. +} diff --git a/man/air_update_description_table.Rd b/man/air_update_description_table.Rd new file mode 100644 index 0000000..6ea947d --- /dev/null +++ b/man/air_update_description_table.Rd @@ -0,0 +1,49 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_update_description_table} +\alias{air_update_description_table} +\title{Update the description table} +\usage{ +air_update_description_table( + base, + description, + table_name = "Description", + join_field = "title", + record_id_field = "id" +) +} +\arguments{ +\item{base}{String. Base id} + +\item{description}{Data frame. Contains updated description} + +\item{table_name}{String. Name of description table} + +\item{join_field}{String. Field to perform join on} + +\item{record_id_field}{String. Name of the record id field} +} +\value{ +list that logs updates +} +\description{ +Update the descriptive metadata table in airtable +} +\examples{ + +\dontrun{ + +base <- "appXXXXXXXX" +table_name <- "Description" +# get description from table +description <- air_get_base_description_from_table(base, table_name) +# update the identifier field +description$identifier <- "fake.doi.xyz/029940" +# update the table +air_update_description_table(base,description) + + +} + + +} diff --git a/man/air_update_field.Rd b/man/air_update_field.Rd new file mode 100644 index 0000000..4caafec --- /dev/null +++ b/man/air_update_field.Rd @@ -0,0 +1,93 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_metadata_api.R +\name{air_update_field} +\alias{air_update_field} +\title{Update a field name and/or description} +\usage{ +air_update_field(base, table_id, field_id, name = NULL, description = NULL) +} +\arguments{ +\item{base}{String. Base id} + +\item{table_id}{String. ID for table that contains the field to be updated} + +\item{field_id}{String. ID of field to be updated} + +\item{name}{String. updated name (optional)} + +\item{description}{String. updated description (option)} +} +\value{ +List. Describes the changes that happened to the field +} +\description{ +Must update either the name or the description. +See "https://airtable.com/developers/web/api/update-field" for more details. +} +\examples{ +\dontrun{ +base <- "appVjIfAo8AJlfTkx" + +schema <- air_get_schema("appVjIfAo8AJlfTkx") + +table_id <- schema$tables[1,c("id")] + +field_id <- schema$tables$fields[[1]][2,]$id + +## update name and description + +name <- "New Name" + +description <- "Updated Description" + +out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +name = name, description = description) + +### just name + +name <- "New New Name" + +out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +name = name) + + +## just description + +description <- "Better description" + +out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +description = description) + +## set name to number + +name <- 1234 + +out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +name = name) + + +# set description to number + +description <- 1234 + +out <- air_update_field(base = base,table_id = table_id,field_id = field_id, +description = description) + +# bulk update names and descriptions from a data frame + +field_ids <- schema$tables$fields[[1]]$id + +field_names <- sprintf("\%s_bulk_update",schema$tables$fields[[1]]$name) + +field_descriptions <- sprintf("\%s BULK UPDATE", +schema$tables$fields[[1]]$description) + +df <- data.frame("field_id"= field_ids,"name"=field_names, +"description"=field_descriptions) + +purrr::pmap(df,function(field_id,name,description){ + air_update_field(base = base,table_id = table_id,field_id = field_id, + name = name, description = description) +}) +} +} diff --git a/man/air_update_metadata_table.Rd b/man/air_update_metadata_table.Rd new file mode 100644 index 0000000..0015337 --- /dev/null +++ b/man/air_update_metadata_table.Rd @@ -0,0 +1,39 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{air_update_metadata_table} +\alias{air_update_metadata_table} +\title{Update the structural metadata table} +\usage{ +air_update_metadata_table( + base, + meta_data, + table_name = "Meta Data", + join_field = "field_id", + record_id_field = "id" +) +} +\arguments{ +\item{base}{String. Base id} + +\item{meta_data}{Data frame. Contains metadata records. From air_generate_metadata*} + +\item{table_name}{String. Name of metadata table} + +\item{join_field}{String. Name of field to join new and current metadata. Likely \code{field_id}} + +\item{record_id_field}{String. Name of record id field. Like \code{id}} +} +\value{ +List. Log of results for updating metadata +} +\description{ +Update the structural metadata table +} +\examples{ +\dontrun{ +base = "appVjIfAo8AJlfTkx" +metadata <- air_generate_metadata_from_api(base = base) +air_update_metadata_table(base,metadata) +} + +} diff --git a/man/airtabler-package.Rd b/man/airtabler-package.Rd index aae9d1e..3b61112 100644 --- a/man/airtabler-package.Rd +++ b/man/airtabler-package.Rd @@ -16,17 +16,24 @@ Provides access to the Airtable API (\url{http://airtable.com/api}). \section{API key}{ - Generate the airtable API key from your Airtable account page - (http://airtable.com/account). + Generate the Airtable API token from your Airtable account page + (http://airtable.com/create/tokens). \pkg{airtabler} functions will read the API key from environment variable \code{AIRTABLE_API_KEY}. To start R session with the - initialized environvent variable create an \code{.Renviron} file in your R home + initialized environment variable create an \code{.Renviron} file in your R home with a line like this: \code{AIRTABLE_API_KEY=************} To check where your R home is, try \code{normalizePath("~")}. + + The \pkg{usethis} and \pkg{dotenv} packages are useful for setting environment + variables. + \code{usethis::edit_r_environ} allow you to modify the \code{.Renviron} file. + \code{dotenv::load_dot_env} allows you to load environment variables from a + \code{.env} file. This second approach is especially helpful if you work + with multiple tokens. } \section{Usage}{ diff --git a/man/fetch_all.Rd b/man/fetch_all.Rd new file mode 100644 index 0000000..62e0d1f --- /dev/null +++ b/man/fetch_all.Rd @@ -0,0 +1,56 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/fetch_all.R +\name{fetch_all} +\alias{fetch_all} +\title{Fetch All Records in an Airtable} +\usage{ +fetch_all(base, table_name, ...) +} +\arguments{ +\item{base}{String. ID for the base or app to be fetched} + +\item{table_name}{String. Name of the table to be fetched from the base} + +\item{...}{Additional arguments to pass to [air_get()]. \code{view} is a +commonly used additional argument.} +} +\value{ +dataframe +} +\description{ +Airtable limits the number of records that can be pulled from a base to 100. +This function pulls records based on a query then checks if there is an +offset value. While there is an offset value, it uses that value to generate +the next query, thus moving down the records until all records have been +fetched from the database. +} +\section{Adding airtable API key to your environment}{ + +Airtable requires an api key to fetch data. +Generate the airtable API key from your [Airtable account](http://airtable.com/account) page. + +\code{airtabler} functions will read the API key from +environment variable \code{AIRTABLE_API_KEY}. To start R session with the +initialized environment variable create an \code{.Renviron} file in your +home directory with a line like this: + +\code{AIRTABLE_API_KEY=your_api_key_here} + +You can use \code{usethis::edit_r_environ()} to open and edit your .Renviron +file. + +Also consider using the `dotenv` package with a .env file for storing +sensitive variables. Remember add to.gitignore or encrypt the .env file to +avoid sharing sensitive variables. +} + +\examples{ +# Each base has a fully described API +# app_id <- "appVjIfAo8AJlfTkx" # ID for the base we are fetching. +# Note that you can pass a `view` argument to air_get or fetch_all to get only +# a view of a table (say, only validated records, or some other filtered view), +# e.g., +# bats <- fetch_all(app_id, "images", view = "Status View") +# talks <- fetch_all(app_id, "images") + +} diff --git a/man/fetch_all_json.Rd b/man/fetch_all_json.Rd new file mode 100644 index 0000000..5432243 --- /dev/null +++ b/man/fetch_all_json.Rd @@ -0,0 +1,32 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_get_json.R +\name{fetch_all_json} +\alias{fetch_all_json} +\title{Get the full outputs of a table as single json object} +\usage{ +fetch_all_json(base, table_name, ...) +} +\arguments{ +\item{base}{String. Base ID} + +\item{table_name}{String. Table name} + +\item{...}{additional parameters to pass to air_get_json} +} +\value{ +json as string +} +\description{ +Get the full outputs of a table as single json object +} +\examples{ + +\dontrun{ +base <- "appXXXXXXX" +table_name <- "My Table" + +fetch_all_json(base, table_name) + +} + +} diff --git a/man/flatten_col_to_chr.Rd b/man/flatten_col_to_chr.Rd new file mode 100644 index 0000000..7d2dbf6 --- /dev/null +++ b/man/flatten_col_to_chr.Rd @@ -0,0 +1,42 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{flatten_col_to_chr} +\alias{flatten_col_to_chr} +\title{Flatten list columns to character} +\usage{ +flatten_col_to_chr(data_frame) +} +\arguments{ +\item{data_frame}{a data frame, tibble or other data frame like object} +} +\value{ +data_frame with list columns converted to character vectors. +} +\description{ +Similar in spirit to purrr::flatten_chr except +that it can handle NULL values in lists and returns outputs +that can be written to csv. +} +\details{ +Because the outputs are intended for use in CSV files, we must use +double quotes to indicate that the commas separating list values do +not delimit cells. This conforms to RFC 4180 standard for CSVs. +\url{https://datatracker.ietf.org/doc/html/rfc4180} +} +\examples{ + +data_frame <- data.frame(a = I(list(list("Hello"), +list("Aloha"), +NULL, +list("Hola","Bonjour","Merhaba") +)), +b = 1:4, +c = letters[1:4], +d = I(data.frame(id = 1:4, name = "bob", email = "bob@example.com")) +) + +test_df <- flatten_col_to_chr(data_frame) + +str(test_df) + +} diff --git a/man/get_unique_field_values.Rd b/man/get_unique_field_values.Rd new file mode 100644 index 0000000..8758aa7 --- /dev/null +++ b/man/get_unique_field_values.Rd @@ -0,0 +1,26 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/get_unique_field_values.R +\name{get_unique_field_values} +\alias{get_unique_field_values} +\title{Get unique values from a field} +\usage{ +get_unique_field_values(base, table_name, fields) +} +\arguments{ +\item{base}{String. ID of airtable base} + +\item{table_name}{String. Name of table in base} + +\item{fields}{List. Names of fields} +} +\value{ +vector of unique values +} +\description{ +Because the airtable api lacks an easy solution for getting +unique values from a field in a table, we have this function +which accepts a list of fields and returns their unique +values. Currently, if multiple fields are listed, all unique +values from all fields will be returned in a single vector. +This may change in future iterations. +} diff --git a/man/read_excel_url.Rd b/man/read_excel_url.Rd new file mode 100644 index 0000000..55c25e9 --- /dev/null +++ b/man/read_excel_url.Rd @@ -0,0 +1,23 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/read_excel_url.R +\name{read_excel_url} +\alias{read_excel_url} +\title{Read an excel file from URL} +\usage{ +read_excel_url(url, fileext = ".xslx", parse_all_sheets = FALSE, ...) +} +\arguments{ +\item{url}{String. Url for file} + +\item{fileext}{String. File extension for temp file} + +\item{parse_all_sheets}{Logical. Should all sheets be parsed?} + +\item{...}{additional arguments to pass to \code{read_excel}} +} +\value{ +tibble or list of tibbles if parse_all_sheets = TRUE +} +\description{ +Extends \code{readxl::read_excel} to allow for reading from a URL. +} diff --git a/man/set_diff.Rd b/man/set_diff.Rd new file mode 100644 index 0000000..8d71da6 --- /dev/null +++ b/man/set_diff.Rd @@ -0,0 +1,35 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/air_dump.R +\name{set_diff} +\alias{set_diff} +\title{Get items that differ between x and y} +\usage{ +set_diff(x, y) +} +\arguments{ +\item{x}{a set of values.} + +\item{y}{a set of values.} +} +\value{ +Unique values from X and Y, NULL if no unique values. +} +\description{ +Unlike setdiff, this function creates the union of x and y then +removes values that are in the intersect, providing values +that are unique to X and values that are unique to Y. +} +\examples{ +a <- 1:3 +b <- 2:4 + +set_diff(a,b) +# returns 1,4 + +x <- 1:3 +y <- 1:3 + +set_diff(x,y) +# returns NULL + +} diff --git a/readme.md b/readme.md index 522506a..87cadf7 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,9 @@ -# airtabler +--- +title: "airtabler" +output: + html_document: + keep_md: yes +--- Provides access to the [Airtable API](http://airtable.com/api) @@ -7,7 +12,7 @@ Provides access to the [Airtable API](http://airtable.com/api) ```r -devtools::install_github("bergant/airtabler") +devtools::install_github("One-Health-Research-Consulting/airtabler") ``` ## Setup @@ -15,18 +20,38 @@ devtools::install_github("bergant/airtabler") graphical interface, your Airtable base will provide its own API to create, read, update, and destroy records. - [airtable.com/api](http://airtable.com/api) -## Get and store the API key -Generate the airtable API key from your [Airtable account](http://airtable.com/account) page. +## Get and store the API tokens -__airtabler__ functions will read the API key from +** As of November 2022 Airtable recommends using scoped tokens. The personal access tokens can be used interchangeably with the now superseded Airtable API Key. + +Create appropriately [scoped personal access tokens](https://airtable.com/developers/web/guides/personal-access-tokens). + +__airtabler__ functions will read the API token from environment variable `AIRTABLE_API_KEY`. To start R session with the - initialized environvent variable create an `.Renviron` file in your home directory - with a line like this: + initialized environvent variable create an `.Renviron` file in your home directory. -`AIRTABLE_API_KEY=your_api_key_here` +```r +usethis::edit_r_environ +``` + +In .Renviron add the following: +``` +AIRTABLE_API_KEY=your_api_token_here + +``` + +**NOTE: Be sure the last line of your .Renviron file is an empty return line** To check where your home is, type `path.expand("~")` in your R console. +If you're frequently working across multiple bases, consider using [`gitcrypt`](https://ecohealthalliance.github.io/eha-ma-handbook/16-encryption.html#set-up-encryption-for-a-repo-that-did-not-previously-use-git-crypt.) and the [`dotenv` package](https://cran.r-project.org/web/packages/dotenv/dotenv.pdf) to securely manage multiple tokens. + + +### Using the metadata API + +In order to use the metadata API, a personal access token or OAuth integration +must be used. User API keys are not supported. See the [airtable guide](https://airtable.com/developers/web/api/get-base-schema) for more +information. ## Usage @@ -55,18 +80,6 @@ hotels <- knitr::kable(hotels[, c("id","Name", "Stars", "Price/night")], format = "markdown") ``` - - -|id |Name |Stars | Price/night| -|:-----------------|:------------------------------------------------------------|:-----|-----------:| -|reccPOcMQaYt1tthb |Heritage Christchurch Hotel (Christchurch, New Zealand) |**** | 176| -|receHGZJ22WyUxocl |Urikana Boutique Hotel (Teresopolis, Brazil) |***** | 146| -|recgKO7K15YyWEsdb |Radisson Blu Hotel Marseille Vieux Port (Marseilles, France) |**** | 170| -|recjJJ4TX38sUwzfj |Hotel Berg (Keflavík, Iceland) |*** | 136| -|recjUU2GT28yVvw7l |Sheraton Nha Trang (Nha Trang, Vietnam) |***** | 136| -|reckPH6G384y3suac |Grand Residences Riviera Cancun (Puerto Morelos, Mexico) |***** | 278| -|reclG7Bd2g5Dtiw4J |Grand Budapest Hotel (Zubrowka) |***** | 156| - Filter records with formula (see [formula field reference ](https://support.airtable.com/hc/en-us/articles/203255215-Formula-Field-Reference)). @@ -77,17 +90,6 @@ hotels <- knitr::kable(hotels[, c("id","Name", "Stars", "Avg Review", "Price/night")], format = "markdown") ``` - - -|id |Name |Stars | Avg Review| Price/night| -|:-----------------|:--------------------------------------------------------|:-----|----------:|-----------:| -|reccPOcMQaYt1tthb |Heritage Christchurch Hotel (Christchurch, New Zealand) |**** | 8.8| 176| -|receHGZJ22WyUxocl |Urikana Boutique Hotel (Teresopolis, Brazil) |***** | 9.0| 146| -|recjJJ4TX38sUwzfj |Hotel Berg (Keflavík, Iceland) |*** | 9.2| 136| -|recjUU2GT28yVvw7l |Sheraton Nha Trang (Nha Trang, Vietnam) |***** | 8.8| 136| -|reckPH6G384y3suac |Grand Residences Riviera Cancun (Puerto Morelos, Mexico) |***** | 9.1| 278| -|reclG7Bd2g5Dtiw4J |Grand Budapest Hotel (Zubrowka) |***** | 9.0| 156| - Sort data with sort parameter: ```r @@ -101,18 +103,6 @@ hotels <- knitr::kable(hotels[, c("id","Name", "Stars", "Avg Review", "Price/night")], format = "markdown") ``` - - -|id |Name |Stars | Avg Review| Price/night| -|:-----------------|:------------------------------------------------------------|:-----|----------:|-----------:| -|recjJJ4TX38sUwzfj |Hotel Berg (Keflavík, Iceland) |*** | 9.2| 136| -|reckPH6G384y3suac |Grand Residences Riviera Cancun (Puerto Morelos, Mexico) |***** | 9.1| 278| -|receHGZJ22WyUxocl |Urikana Boutique Hotel (Teresopolis, Brazil) |***** | 9.0| 146| -|reclG7Bd2g5Dtiw4J |Grand Budapest Hotel (Zubrowka) |***** | 9.0| 156| -|recjUU2GT28yVvw7l |Sheraton Nha Trang (Nha Trang, Vietnam) |***** | 8.8| 136| -|reccPOcMQaYt1tthb |Heritage Christchurch Hotel (Christchurch, New Zealand) |**** | 8.8| 176| -|recgKO7K15YyWEsdb |Radisson Blu Hotel Marseille Vieux Port (Marseilles, France) |**** | 8.2| 170| - ### Using page size and offset Define page size with `pageSize`: @@ -122,10 +112,6 @@ hotels <- TravelBucketList$Hotels$select(pageSize = 3) nrow(hotels) ``` -``` -## [1] 3 -``` - Continue at offset, returned by previous select: ```r @@ -133,10 +119,6 @@ hotels <- TravelBucketList$Hotels$select(offset = get_offset(hotels)) nrow(hotels) ``` -``` -## [1] 4 -``` - To fetch all rows (even > 100 records) use `select_all`. The `select_all` function will handle the offset and return the result as a single object. @@ -147,10 +129,6 @@ hotels <- TravelBucketList$Hotels$select_all() nrow(hotels) ``` -``` -## [1] 7 -``` - Other optional arguments: @@ -170,19 +148,6 @@ radisson <- str(radisson$fields, max.level = 1) ``` -``` -## List of 9 -## $ Listing URL: chr "https://www.booking.com/hotel/fr/radisson-sas-marseille-vieux-port.html" -## $ Name : chr "Radisson Blu Hotel Marseille Vieux Port (Marseilles, France)" -## $ Price/night: int 170 -## $ Amenities : chr [1:4] "Pool" "Gym" "Restaurant" "Wifi" -## $ Notes : chr "Rooms with African or Provencál decor." -## $ Country : chr "recmSV4PR9ZCWyrk8" -## $ Pictures :'data.frame': 4 obs. of 6 variables: -## $ Stars : chr "****" -## $ Avg Review : num 8.2 -``` - ### Insert a record Insert a new record with `insert` function (API returns all record data - including new record ID): @@ -201,10 +166,6 @@ new_hotel <- cat("Inserted a record with ID=", new_hotel$id, sep = "") ``` -``` -## Inserted a record with ID=recGtWMprUr7f2EvT -``` - ### Update a record Update the price of the new hotel (API returns all record data): @@ -223,24 +184,12 @@ cat("Updated a record with ID=", new_hotel$id, ". ", "New price: ", new_hotel$fields$`Price/night`, sep = "") ``` -``` -## Updated a record with ID=recGtWMprUr7f2EvT. New price: 120 -``` - ### Delete a record ```r TravelBucketList$Hotels$delete(new_hotel$id) ``` -``` -## $deleted -## [1] TRUE -## -## $id -## [1] "recGtWMprUr7f2EvT" -``` - ## Working with data frames diff --git a/readme.rmd b/readme.rmd index 9f6ddeb..ad7f16a 100644 --- a/readme.rmd +++ b/readme.rmd @@ -7,13 +7,13 @@ output: Provides access to the [Airtable API](http://airtable.com/api) ```{r setup, include=FALSE} -knitr::opts_chunk$set(echo = TRUE) +knitr::opts_chunk$set(echo = TRUE, eval = FALSE) ``` ## Install ```{r install, eval=FALSE} -devtools::install_github("bergant/airtabler") +devtools::install_github("ecohealthalliance/airtabler") ``` ## Setup @@ -21,18 +21,38 @@ devtools::install_github("bergant/airtabler") graphical interface, your Airtable base will provide its own API to create, read, update, and destroy records. - [airtable.com/api](http://airtable.com/api) -## Get and store the API key -Generate the airtable API key from your [Airtable account](http://airtable.com/account) page. +## Get and store the API tokens -__airtabler__ functions will read the API key from +** As of November 2022 Airtable recommends using scoped tokens. The personal access tokens can be used interchangeably with the now superseded Airtable API Key. + +Create appropriately [scoped personal access tokens](https://airtable.com/developers/web/guides/personal-access-tokens). + +__airtabler__ functions will read the API token from environment variable `AIRTABLE_API_KEY`. To start R session with the - initialized environvent variable create an `.Renviron` file in your home directory - with a line like this: + initialized environvent variable create an `.Renviron` file in your home directory. -`AIRTABLE_API_KEY=your_api_key_here` +```r +usethis::edit_r_environ +``` + +In .Renviron add the following: +``` +AIRTABLE_API_KEY=your_api_token_here + +``` + +**NOTE: Be sure the last line of your .Renviron file is an empty return line** To check where your home is, type `path.expand("~")` in your R console. +If you're frequently working across multiple bases, consider using [`gitcrypt`](https://ecohealthalliance.github.io/eha-ma-handbook/16-encryption.html#set-up-encryption-for-a-repo-that-did-not-previously-use-git-crypt.) and the [`dotenv` package](https://cran.r-project.org/web/packages/dotenv/dotenv.pdf) to securely manage multiple tokens. + + +### Using the metadata API + +In order to use the metadata API, a personal access token or OAuth integration +must be used. User API keys are not supported. See the [airtable guide](https://airtable.com/developers/web/api/get-base-schema) for more +information. ## Usage diff --git a/vignettes/.gitignore b/vignettes/.gitignore new file mode 100644 index 0000000..097b241 --- /dev/null +++ b/vignettes/.gitignore @@ -0,0 +1,2 @@ +*.html +*.R diff --git a/vignettes/Generate-Metadata-and-Backup.Rmd b/vignettes/Generate-Metadata-and-Backup.Rmd new file mode 100644 index 0000000..672eafb --- /dev/null +++ b/vignettes/Generate-Metadata-and-Backup.Rmd @@ -0,0 +1,245 @@ +--- +title: "Generate-Metadata-and-Backup" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Generate-Metadata-and-Backup} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>" +) +``` + +```{r setup} +library(airtabler) +``` + +This vignette will demonstrate how to generate and update metadata for an Airtable +base using the [Airtable metadata api](https://airtable.com/developers/web/api/get-base-schema) then export the base to create an offline backup. + +*Note* Make sure that you have the [environment variable](https://github.com/gaborcsardi/dotenv) `AIRTABLE_API_KEY` set with your [Airtable API token](https://airtable.com/create/tokens). Make sure your token has schema read and write priveleges. + +## Creating metadata from schema + +`air_generate_metadata_from_api` creates a data frame that holds the structural metadata for your base. Structural metadata describes how entities relate to one another, in this case, how fields relate to tables within the base. + + +```{r generate metadata, eval=FALSE} +# set your base id +base = "appVjIfAo8AJlfTkx" +## create a list +api_metadata <- air_generate_metadata_from_api(base = "appVjIfAo8AJlfTkx") +str(api_metadata) +``` + +## Adding a metadata table to the base + +```{r add metadata table, eval=FALSE} +status <- air_create_metadata_table(base = base,meta_data = api_metadata) +``` + +## Create a descriptive metadata table for the base + +The `description` table will describe who created the base, when, and why. +Descriptive metadata is important for long term storage and reuse. The default +terms in the `description` table match core DCMI terms. Additional terms can be +added. + +```{r descriptive metadata, eval=FALSE} +## create a basic descriptive metadata table + +description <- air_generate_base_description(title = "Base Title", + creator = "Arkady Darell", + created = "2023-04-03", + description = "This base is an example for airtabler") + + +# We could have provided keywords as comma separated values but lets make things +# more interesting by presenting them as a vector +description_with_keywords <- air_generate_base_description(title = "Base Title", + creator = "Arkady Darell", + created = "2023-04-03", + description = "This base is an example for airtabler", + keywords = list(c("Example","R","Package","Airtable"))) + +``` + +## Add a description table to the base + +```{r add descriptive metadata, eval=FALSE} + +## add our vanilla description table - preferred way to work with description data +air_create_description_table(base = base,description = description) + + +## add our description with keywords and - we want keywords to be multiple select +## Note this is not the preferred method but it is possible + +length(names(description_with_keywords)) + +create_choices <- function(x){ + + choice_list <- list() + + choice_list$choices <- purrr::map(x,function(x_item){ + list(name = x_item) + }) + + return(list(choice_list)) # wrap in an extra list to +} + +keyword_options <- create_choices(description_with_keywords$keywords[[1]]) + +air_create_description_table(base = base,description = description_with_keywords, + type = c(rep("singleLineText",9),"multipleSelects"), + options = c(rep(NA,9),keyword_options)) + + +``` + +## Updating metadata tables + +Your data will change so your metadata should update as well. + +### Structural Metadata +```{r update structural metadata, eval=FALSE} + +# get your metadata from the api + metadata <- air_generate_metadata_from_api(base = base) + +# run the function +update_log <- air_update_metadata_table(base,metadata) + +# the update log provides an overview of records that were updated, inserted, or deleted +# and fields that were created in the event that the structure of your metadata table +# changed. + +``` + +### Descriptive Metadata + + + +```{r update descriptive metadata, eval=FALSE} + +base_description <- air_get_base_description_from_table(base = base,table_name = "Description", + field_names_to_snakecase = FALSE) + +base_description$description <- "Keep on updating" + +## since the field types are already established, its a little easier to add multipleSelect keywords +base_description$keywords[[1]] <- append("New Keyword",base_description$keywords[[1]]) + +base_description <- base_description |> + dplyr::select(-createdTime) + +## if your base_description obj has a record id field, use that for the join. Default is title +air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id") + +``` + + +## Using the metadata to create a backup + +Now that you have your metadata tables setup, lets see how we can use them to +create backups. The workflow pulls data down from airtable into R where it can +be written to whatever format you like. There is a built in function for writing +to versioned to CSVs. This is the recommended workflow for creating backups. + +### Pull all data into R + +```{r dump to R, eval=FALSE} + +## pull down the metadata table from airtable +metadata <- air_get_metadata_from_table(base = base,table_name = "Meta Data",add_id_field = FALSE) + +airtable_base <- air_dump(base = base,metadata = metadata,description = base_description) + +summary(airtable_base) + +``` + +### Create a set of versioned CSVs + +This function creates a folder with a unique ID based on a hash of the list +passed to table_list. If the data in your base do not change, then the hash +won't change and no new version of the data will be written. + +```{r dump to csv, eval=FALSE} + +# dump to csv +air_dump_to_csv(table_list = airtable_base) + +# Make a change to the description table to show how hashing works +base_description <- air_get_base_description_from_table(base = base,table_name = "Description", + field_names_to_snakecase = FALSE) + +## since the field types are already established, its a little easier to add multipleSelect keywords +base_description$keywords[[1]] <- append("How will the hash change?",base_description$keywords[[1]]) + +base_description <- base_description |> + dplyr::select(-createdTime) + +## if your base_description obj has a record id field, use that for the join. Default is title +air_update_description_table(base = base,description = base_description,table_name = "Description",join_field = "id") + +## pull the changed based down +airtable_base_updated <- air_dump(base = base,metadata = metadata,description = base_description) + +## dump to csv +air_dump_to_csv(table_list = airtable_base_updated) +``` + +## Backup all bases in a workspace + +*Note* This routine requires personal access tokens that can read and write +data and schema to all bases in a workspace. + +Here we are going to take advantage of the [`list bases`](https://airtable.com/developers/web/api/list-bases) +endpoint in the airtable API. It will list all bases that a token has access to. + +```{r workspace backup, eval=FALSE} + +# get all bases associated with token +bases <- air_list_bases() + +# generate the description for our second +description_2 <- air_generate_base_description(title = bases$bases$name[[2]],creator = "Arkady Darell",created = Sys.Date(),description = "A base to demo bulk backups") + +description_log <- air_create_description_table(bases$bases$id[[2]],description = description_2) + +metadata_2 <- air_generate_metadata_from_api(bases$bases$id[[2]]) + +metadata_log <- air_create_metadata_table(bases$bases$id[[2]],meta_data = metadata_2) + +## add metadata and descriptive data to bases list + +base_descriptions <- purrr::map(bases$bases$id,function(base){ + air_get_base_description_from_table(base,table_name = "Description") +}) + + +base_metadata <- purrr::map(bases$bases$id,function(base){ + air_get_metadata_from_table(base,table_name = "Meta Data") +}) + + +bases$bases$description <- base_descriptions +bases$bases$metadata <- base_metadata + +base_df <- bases$bases + +for(i in 1:nrow(base_df)){ + base_item <- base_df[i,] + table_list <- air_dump(base = base_item$id ,metadata = base_item$metadata[[1]],description = base_item$description[[1]]) + + output_dir <- sprintf("Airtabler Workspace/%s", base_item$name) + air_dump_to_csv(table_list,output_dir = output_dir) +} + + +```