RootPath <- paste0("/your/path/here")

# Create the working directory in the RootPath if it doesn't exist already
if (!dir.exists(paste0(RootPath, "/Data_acquisition_workflow"))) {
    dir.create(paste0(RootPath, "/Data_acquisition_workflow"), recursive = TRUE)
}
# Set the working directory
setwd(paste0(RootPath, "/Data_acquisition_workflow"))

renv::activate(project = paste0(RootPath, "/Data_acquisition_workflow"))

if (!require("BiocManager", quietly = TRUE)) install.packages("BiocManager", repos = "http://cran.us.r-project.org")

BiocManager::install("ComplexHeatmap")

# Install remotes if needed
if (!require("remotes", quietly = TRUE)) install.packages("remotes", repos = "http://cran.us.r-project.org")
# Download and then load rnaturalearthhires
remotes::install_github("ropensci/rnaturalearthhires")
install.packages("rnaturalearthhires", repos = "https://ropensci.r-universe.dev",
    type = "source")
library(rnaturalearthhires)

install.packages("BeeBDC")
library(BeeBDC)

renv::snapshot(project = paste0(RootPath, "/Data_acquisition_workflow"), prompt = FALSE)
## The following package(s) will be updated in the lockfile:
## 
## # RSPM -----------------------------------------------------------------------
## - renv   [* -> 1.0.11]
## 
## The version of R recorded in the lockfile will be updated:
## - R      [* -> 4.4.2]
## 
## - Lockfile written to "/tmp/RtmpIYpfHt/Data_acquisition_workflow/renv.lock".

BeeBDC::dirMaker(RootPath = RootPath, RDoc = "vignettes/BeeBDC_main.Rmd") %>%
    # Add paths created by this function to the environment()
list2env(envir = parent.env(environment()))

lapply(c("ComplexHeatmap", "magrittr"), library, character.only = TRUE)
## Loading required package: grid
## 
## Attaching package: 'grid'
## The following object is masked from 'package:terra':
## 
##     depth
## ========================================
## ComplexHeatmap version 2.23.0
## Bioconductor page: http://bioconductor.org/packages/ComplexHeatmap/
## Github page: https://github.com/jokergoo/ComplexHeatmap
## Documentation: http://jokergoo.github.io/ComplexHeatmap-reference
## 
## If you use it in published research, please cite either one:
## - Gu, Z. Complex Heatmap Visualization. iMeta 2022.
## - Gu, Z. Complex heatmaps reveal patterns and correlations in multidimensional 
##     genomic data. Bioinformatics 2016.
## 
## 
## The new InteractiveComplexHeatmap package can directly export static 
## complex heatmaps into an interactive Shiny app with zero effort. Have a try!
## 
## This message can be suppressed by:
##   suppressPackageStartupMessages(library(ComplexHeatmap))
## ========================================
## 
## Attaching package: 'ComplexHeatmap'
## The following object is masked from 'package:terra':
## 
##     draw
## The following object is masked from 'package:R.utils':
## 
##     draw

check_pf <- bdc::bdc_scientificName_empty(data = db_standardized, sci_name = "scientificName")
## 
## bdc_scientificName_empty:
## Flagged 0 records.
## One column was added to the database.
# now that this is saved, remove it to save space in memory
rm(db_standardized)

check_pf <- bdc::bdc_coordinates_empty(data = check_pf, lat = "decimalLatitude",
    lon = "decimalLongitude")
## 
## bdc_coordinates_empty:
## Flagged 42 records.
## One column was added to the database.

check_pf <- bdc::bdc_coordinates_outOfRange(data = check_pf, lat = "decimalLatitude",
    lon = "decimalLongitude")
## 
## bdc_coordinates_outOfRange:
## Flagged 0 records.
## One column was added to the database.

check_pf <- bdc::bdc_basisOfRecords_notStandard(
  data = check_pf,
  basisOfRecord = "basisOfRecord",
  names_to_keep = c(
    # Keep all plus some at the bottom.
    "Event",
    "HUMAN_OBSERVATION",
    "HumanObservation",
    "LIVING_SPECIMEN",
    "LivingSpecimen",
    "MACHINE_OBSERVATION",
    "MachineObservation",
    "MATERIAL_SAMPLE",
    "O",
    "Occurrence",
    "MaterialSample",
    "OBSERVATION",
    "Preserved Specimen",
    "PRESERVED_SPECIMEN",
    "preservedspecimen Specimen",
    "Preservedspecimen",
    "PreservedSpecimen",
    "preservedspecimen",
    "S",
    "Specimen",
    "Taxon",
    "UNKNOWN",
    "",
    NA,
    "NA",
    "LITERATURE", 
    "None", "Pinned Specimen", "Voucher reared", "Emerged specimen"
  ))
## 
## bdc_basisOfRecords_notStandard:
## Flagged 1 of the following specific nature:
##  MATERIAL_CITATION 
## One column was added to the database.

  # Standardise country names and add ISO2 codes if needed
check_pf <- bdc::bdc_country_standardized(
  # Remove the countryCode and country_suggested columns to avoid an error with 
    # where two "countryCode" and "country_suggested" columns exist (i.e. if the dataset has been  
    # run before)
  data = check_pf %>% dplyr::select(!tidyselect::any_of(c("countryCode", "country_suggested"))),
  country = "country"
) 
## Loading auxiliary data: country names
## Standardizing country names
## country found: Argentina
## country found: Australia
## country found: Belgium
## country found: Brazil
## country found: Canada
## country found: Colombia
## country found: Costa Rica
## country found: Ecuador
## country found: Estonia
## country found: Finland
## country found: France
## country found: Germany
## country found: Ireland
## country found: Mexico
## country found: Norway
## country found: South Africa
## country found: Sweden
## country found: Switzerland
## 
## bdc_country_standardized:
## The country names of 5 records were standardized.
## Two columns ('country_suggested' and 'countryCode') were added to the database.

check_pf <- BeeBDC::jbd_Ctrans_chunker(
  # bdc_coordinates_transposed inputs
  data = check_pf,
  id = "database_id",
  lat = "decimalLatitude",
  lon = "decimalLongitude",
  country = "country",
  countryCode = "countryCode",
  border_buffer = 0.2, # in decimal degrees (~22 km at the equator)
  save_outputs = TRUE,
  sci_names = "scientificName",
  # chunker inputs
  stepSize = 1000000,  # How many rows to process at a time
  chunkStart = 1,  # Start row
  append = FALSE,  # If FALSE it may overwrite existing dataset
  progressiveSave = FALSE,
    # In a normal run, please use scale = "large"
  scale = "medium",
  path = OutPath_Check,
  mc.cores = 1
) 
##  - Running chunker with:
## stepSize = 1,000,000
## chunkStart = 1
## chunkEnd = 1,000,000
## append = FALSE
##  - Starting chunk 1...
## From 1 to 1,000,000
##  - Finished chunk 1 of 1. Total records examined: 205

table(check_pf$coordinates_transposed, useNA = "always")

check_pf %>%
    readr::write_excel_csv(., paste(OutPath_Intermediate, "01_prefilter_database.csv",
        sep = "/"))

if (!exists("check_pf")) {
    check_pf <- readr::read_csv(paste(OutPath_Intermediate, "01_prefilter_database.csv",
        sep = "/"), col_types = BeeBDC::ColTypeR())
}

check_pf <- BeeBDC::jbd_coordCountryInconsistent(data = check_pf, lon = "decimalLongitude",
    lat = "decimalLatitude", scale = 50, pointBuffer = 0.01)
##  - Downloading naturalearth map...
## Spherical geometry (s2) switched off
##  - Extracting initial country names without buffer...
##  - Buffering naturalearth map by pointBuffer...
## dist is assumed to be in decimal degrees (arc_degrees).
##  - Extracting FAILED country names WITH buffer...
## 
## jbd_coordinates_country_inconsistent:
## Flagged 2 records.
## The column, '.coordinates_country_inconsistent', was added to the database.
##  - Completed in 0.8 secs

check_pf %>%
    readr::write_excel_csv(., paste(OutPath_Intermediate, "01_prefilter_database.csv",
        sep = "/"))

xyFromLocality <- bdc::bdc_coordinates_from_locality(data = check_pf, locality = "locality",
    lon = "decimalLongitude", lat = "decimalLatitude", save_outputs = FALSE)
## 
## bdc_coordinates_from_locality 
## Found 38 records missing or with invalid coordinates but with potentially useful information on locality.

# Save the resultant data
xyFromLocality %>%
    readr::write_excel_csv(paste(OutPath_Check, "01_coordinates_from_locality.csv",
        sep = "/"))

rm(xyFromLocality)

check_pf <- BeeBDC::flagAbsent(data = check_pf, PresAbs = "occurrenceStatus")
## \.occurrenceAbsent:
##  Flagged 8 absent records:
##  One column was added to the database.

check_pf <- BeeBDC::flagLicense(data = check_pf,
                    strings_to_restrict = "all",
                    # DON'T flag if in the following dataSource(s)
                    excludeDataSource = NULL)
## \.unLicensed:
##  Flagged 0 records that may NOT be used.
##  One column was added to the database.

check_pf <- BeeBDC::GBIFissues(data = check_pf, issueColumn = "issue", GBIFflags = c("COORDINATE_INVALID",
    "ZERO_COORDINATE"))
##  - jbd_GBIFissues:
## Flagged 0 
##   The .GBIFflags column was added to the database.

check_pf %>%
    readr::write_excel_csv(., paste(OutPath_Intermediate, "01_prefilter_output.csv",
        sep = "/"))

parse_names <- bdc::bdc_clean_names(sci_names = database$scientificName, save_outputs = FALSE)

parse_names <- parse_names %>%
    dplyr::select(.uncer_terms, names_clean)

database <- dplyr::bind_cols(database)
rm(parse_names)

taxonomyFile <- BeeBDC::beesTaxonomy()

database <- BeeBDC::harmoniseR(path = DataPath, #The path to a folder that the output can be saved
                       taxonomy = taxonomyFile, # The formatted taxonomy file
                       data = database,
                       mc.cores = 1)
##  - Formatting taxonomy for matching...
## The names_clean column was not found and will be temporarily copied from scientificName
## 
##  - Harmonise the occurrence data with unambiguous names...
## 
##  - Attempting to harmonise the occurrence data with ambiguous names...
##  - Formatting merged datasets...
## Removing the names_clean column...
##  - We matched valid names to 196 of 205 occurrence records. This leaves a total of 9 unmatched occurrence records.
## 
## harmoniseR:
## 9
## records were flagged.
## The column, '.invalidName' was added to the database.
##  - We updated the following columns: scientificName, species, family, subfamily, genus, subgenus, specificEpithet, infraspecificEpithet, and scientificNameAuthorship. The previous scientificName column was converted to verbatimScientificName
##  - Completed in 0.27 secs

rm(taxonomyFile)

database %>%
    readr::write_excel_csv(., paste(DataPath, "Output", "Intermediate", "02_taxonomy_database.csv",
        sep = "/"))

flagFile <- BeeBDC::flagRecorder(data = database, outPath = paste(OutPath_Report,
    sep = ""), fileName = paste0("flagsRecorded_", Sys.Date(), ".csv"), idColumns = c("database_id",
    "id", "catalogNumber", "occurrenceID", "dataSource"), append = TRUE, printSummary = TRUE)

check_space <- BeeBDC::jbd_coordinates_precision(data = database, lon = "decimalLongitude",
    lat = "decimalLatitude", ndec = 2  # number of decimals to be tested
)
## jbd_coordinates_precision:
## Flagged 61 records
## The '.rou' column was added to the database.

rm(database)

check_space %>%
    readr::write_excel_csv(., paste(OutPath_Intermediate, "03_space_inter_database.csv",
        sep = "/"))

tempSpace <- check_space %>%
    dplyr::filter(!.coordinates_empty == FALSE) %>%
    dplyr::filter(!.coordinates_outOfRange == FALSE)

tempSpace <-
  CoordinateCleaner::clean_coordinates(
    x =  tempSpace,
    lon = "decimalLongitude",
    lat = "decimalLatitude",
    species = "scientificName",
    countries = NULL, # Tests if coords are from x country. This is not needed.
    tests = c(
      "capitals",     # records within 0.5 km of capitals centroids
      "centroids",    # records within 1 km around country and province centroids
      "equal",      # records with equal coordinates
      "gbif",         # records within 1 km of GBIF headquarters. (says 1 degree in package, but code says 1000 m)
      "institutions", # records within 100m of zoo and herbaria
      "zeros"       # records with coordinates 0,0
      # "seas"        # Not flagged as this should be flagged by coordinate country inconsistent
    ),
    capitals_rad = 1000,
    centroids_rad = 500,
    centroids_detail = "both", # test both country and province centroids
    inst_rad = 100, # remove zoo and herbaria within 100m
    range_rad = 0,
    zeros_rad = 0.5,
    capitals_ref = NULL,
    centroids_ref = NULL,
    country_ref = NULL,
    country_refcol = "countryCode",
    inst_ref = NULL,
    range_ref = NULL,
    # seas_scale = 50,
    value = "spatialvalid" # result of tests are appended in separate columns
  ) %>%
      # Remove duplicate .summary column that can be replaced later and turn into a tibble
  dplyr::select(!tidyselect::starts_with(".summary")) %>%
  dplyr::tibble()

check_space <- tempSpace %>%
    # Re-bind with the records that were removed earlier
dplyr::bind_rows(check_space %>%
    dplyr::filter(.coordinates_empty == FALSE | .coordinates_outOfRange == FALSE))

rm(tempSpace)

check_space %>%
    readr::write_excel_csv(paste(OutPath_Intermediate, "03_space_inter_database.csv",
        sep = "/"))

check_space <- BeeBDC::diagonAlley(
  data = check_space,
  # The minimum number of repeats needed to find a sequence in for flagging
  minRepeats = 6,
  ndec = 3,
  groupingColumns = c("eventDate", "recordedBy", "datasetName"),
  mc.cores = 1)
## Removing rounded coordinates with BeeBDC::jbd_coordinates_precision...
## jbd_coordinates_precision:
## Removed 68 records.
##  - Merging results and adding the .sequential column...
## 
## diagonAlley:
## Flagged 0 records
## The .sequential column was added to the database.
##  - Completed in 0.03 secs

griddingDF <- check_space %>%
    # Exclude NA lat and lon values
tidyr::drop_na(c("decimalLatitude", "decimalLongitude")) %>%
    # Group by the dataset name
dplyr::group_by(datasetName) %>%
    # Remove rows that aren't unique for lat and long
dplyr::distinct(decimalLongitude, decimalLatitude, .keep_all = TRUE) %>%
    # Find the groups with 4 or more occurrence records
dplyr::filter(dplyr::n() >= 4) %>%
    dplyr::ungroup()

gridded_datasets <- CoordinateCleaner::cd_round(x = griddingDF, lon = "decimalLongitude",
    lat = "decimalLatitude", ds = "datasetName", T1 = 7, min_unique_ds_size = 4,
    test = "both", value = "dataset", graphs = FALSE, verbose = TRUE, reg_out_thresh = 2,
    reg_dist_min = 0.1, reg_dist_max = 2) %>%
    dplyr::tibble()
# The griddingDF is no longer needed. remove it.
rm(griddingDF)

check_space <- check_space %>%
  # Join the datasets
  dplyr::left_join(
    # Select the columns of interest
    dplyr::select(gridded_datasets, dataset, lon.flag, lat.flag, summary),
    by = c("datasetName" = "dataset")) %>%
  # Make new columns with more-consistent naming and change the NA vlaues to = TRUE (not flagged)
  dplyr::mutate(.lonFlag = tidyr::replace_na(lon.flag, TRUE),
                .latFlag = tidyr::replace_na(lat.flag, TRUE),
                .gridSummary = tidyr::replace_na(summary, TRUE)) %>%
  # Remove old columns
  dplyr::select(!c(lon.flag, lat.flag, summary))

gridded_datasets %>%
    readr::write_excel_csv(paste(OutPath_Intermediate, "03_space_griddedDatasets.csv",
        sep = "/"))

rm(gridded_datasets)

check_space <- BeeBDC::coordUncerFlagR(data = check_space, uncerColumn = "coordinateUncertaintyInMeters",
    threshold = 1000)
## \coordUncerFlagR:
##  Flagged 33 geographically uncertain records:
##  The column '.uncertaintyThreshold' was added to the database.

checklistFile <- BeeBDC::beesChecklist()

# load in the small test dataset in the background
system.file("extdata", "testChecklist.rda", package = "BeeBDC") |>
    load()
# Rename the file
taxonomyFile <- testChecklist
rm(testChecklist)

check_space <- BeeBDC::countryOutlieRs(checklist = checklistFile,
                        data = check_space,
                        keepAdjacentCountry = TRUE,
                        pointBuffer = 0.05,
                          # Scale of map to return, one of 110, 50, 10 OR 'small', 'medium', 'large'
                          # Smaller numbers will result in much longer calculation times. 
                          # We have not attempted a scale of 10.
                        scale = 50,
                        mc.cores = 1)
## Error: object 'checklistFile' not found

check_space <- BeeBDC::continentOutlieRs(checklist = checklistFile,
                        data = check_space,
                        keepAdjacentContinent = FALSE,
                        pointBuffer = 0.05,
                          # Scale of map to return, one of 110, 50, 10 OR 'small', 'medium', 'large'
                          # Smaller numbers will result in much longer calculation times. 
                          # We have not attempted a scale of 10.
                        scale = 50,
                        mc.cores = 1)
## Error: object 'checklistFile' not found

# A list of failed species-country combinations and their numbers can be output
# here
check_space %>%
    dplyr::filter(.countryOutlier == FALSE) %>%
    dplyr::select(database_id, scientificName, country) %>%
    dplyr::group_by(scientificName) %>%
    dplyr::mutate(count_scientificName = n()) %>%
    dplyr::distinct(scientificName, country, .keep_all = TRUE) %>%
    readr::write_excel_csv(paste(OutPath_Intermediate, "03_space_failedCountryChecklist.csv",
        sep = "/"))

check_space <- BeeBDC::summaryFun(data = check_space, dontFilterThese = NULL, removeFilterColumns = FALSE,
    filterClean = FALSE)

check_space %>%
  dplyr::filter(.summary == FALSE) %>% # map only records flagged as FALSE
  bdc::bdc_quickmap(
    data = .,
    lon = "decimalLongitude",
    lat = "decimalLatitude",
    col_to_map = ".summary",
    size = 0.9
  )

(report <- bdc::bdc_create_report(data = dplyr::tibble(check_space %>%
    dplyr::select(!.uncer_terms)), database_id = "database_id", workflow_step = "space",
    save_report = TRUE))

(figures <- BeeBDC::jbd_create_figures(data = dplyr::tibble(check_space %>%
    dplyr::select(!.uncer_terms)), path = DataPath, database_id = "database_id",
    workflow_step = "space", save_figures = TRUE))

check_space %>%
    readr::write_excel_csv(paste(OutPath_Intermediate, "03_space_inter_database.csv",
        sep = "/"))

BeeBDC::flagRecorder(data = check_space, outPath = paste(OutPath_Report, sep = ""),
    fileName = paste0("flagsRecorded_", Sys.Date(), ".csv"), idColumns = c("database_id",
        "id", "catalogNumber", "occurrenceID", "dataSource"), append = TRUE, printSummary = TRUE)

check_space %>%
    readr::write_excel_csv(., paste(OutPath_Intermediate, "03_space_database.csv",
        sep = "/"))

check_time <- BeeBDC::dateFindR(data = check_time,
                        # Years above this are removed (from the recovered dates only)
                        maxYear = lubridate::year(Sys.Date()),
                        # Years below this are removed (from the recovered dates only)
                        minYear = 1700)
##  - Preparing data...
##  - Extracting dates from year, month, day columns...
##  - Extracting dates from fieldNotes, locationRemarks, and verbatimEventDate columns in unambiguous ymd, dmy, mdy, and my formats...
##  - Extracting year from fieldNotes, locationRemarks, and verbatimEventDate columns in ambiguous formats...
##  - Formating and combining the new data..
##  - Merging all data, nearly there...
##  - Finished. 
## We now have -6 more full eventDate cells than in the input data.
## We modified dates in 
## 174 occurrences.
##  - As it stands, there are 167 complete eventDates and 38 missing dates.
##  - There are also 175 complete year occurrences to filter from. This is up from an initial count of 174 At this rate, you will stand to lose 30 occurrences on the basis of missing year - Operation time: 0.201899766921997 secs

check_time <- bdc::bdc_eventDate_empty(data = check_time, eventDate = "eventDate")
## 
## bdc_eventDate_empty:
## Flagged 38 records.
## One column was added to the database.

check_time <- bdc::bdc_year_outOfRange(data = check_time, eventDate = "year", year_threshold = 1950)
## 
## bdc_year_outOfRange:
## Flagged 21 records.
## One column was added to the database.

check_time <- BeeBDC::summaryFun(
  data = check_time,
  # Don't filter these columns (or NULL)
  dontFilterThese = c(".gridSummary", ".lonFlag", ".latFlag", ".uncer_terms"),
  # Remove the filtering columns?
  removeFilterColumns = FALSE,
  # Filter to ONLY cleaned data?
  filterClean = FALSE)
##  - We will NOT flag the following columns. However, they will remain in the data file.
## .gridSummary, .lonFlag, .latFlag, .uncer_terms
##  - summaryFun:
## Flagged 120 
##   The .summary column was added to the database.

(report <- bdc::bdc_create_report(data = check_time, database_id = "database_id",
    workflow_step = "time", save_report = FALSE))

figures <- BeeBDC::jbd_create_figures(data = check_time, path = DataPath, database_id = "database_id",
    workflow_step = "time", save_figures = TRUE)

figures$year

check_time %>%
    readr::write_excel_csv(., paste(OutPath_Intermediate, "04_time_database.csv",
        sep = "/"))

BeeBDC::flagRecorder(data = check_time, outPath = paste(OutPath_Report, sep = ""),
    fileName = paste0("flagsRecorded_", Sys.Date(), ".csv"), idColumns = c("database_id",
        "id", "catalogNumber", "occurrenceID", "dataSource"), append = TRUE, printSummary = TRUE)

check_time <- BeeBDC::dupeSummary(
  data = check_time,
  path = OutPath_Report,
   # options are "ID","collectionInfo", or "both"
  duplicatedBy = "collectionInfo", 
    # The columns to generate completeness info from (and to sort by completness)
  completeness_cols = c("decimalLatitude",  "decimalLongitude",
                        "scientificName", "eventDate"),
   # The columns to ADDITIONALLY consider when finding duplicates in collectionInfo
  collectionCols = c("decimalLatitude", "decimalLongitude", "scientificName", "eventDate", 
                     "recordedBy"),
    # The columns to combine, one-by-one with the collectionCols
  collectInfoColumns = c("catalogNumber", "otherCatalogNumbers"),
    # Custom comparisons — as a list of columns to compare
     # RAW custom comparisons do not use the character and number thresholds
  CustomComparisonsRAW = dplyr::lst(c("catalogNumber", "institutionCode", "scientificName")),
     # Other custom comparisons use the character and number thresholds
  CustomComparisons = dplyr::lst(c("gbifID", "scientificName"),
                                  c("occurrenceID", "scientificName"),
                                  c("recordId", "scientificName"),
                                  c("id", "scientificName")),
   # The order in which you want to KEEP duplicated based on data source
   # try unique(check_time$dataSource)
  sourceOrder = c("CAES", "Gai", "Ecd","BMont", "BMin", "EPEL", "ASP", "KP", "EcoS", "EaCO",
                  "FSCA", "Bal", "SMC", "Lic", "Arm",
                  "USGS", "ALA", "VicWam", "GBIF","SCAN","iDigBio"),
    # Paige ordering is done using the database_id prefix, not the dataSource prefix.
  prefixOrder = c("Paige", "Dorey"),
    # Set the complexity threshold for id letter and number length
     # minimum number of characters when WITH the numberThreshold
  characterThreshold = 2,
     # minimum number of numbers when WITH the characterThreshold
  numberThreshold = 3,
     # Minimum number of numbers WITHOUT any characters
  numberOnlyThreshold = 5
) %>% # END dupeSummary
  dplyr::as_tibble(col_types = BeeBDC::ColTypeR())
##  - Generating a basic completeness summary from the decimalLatitude, decimalLongitude, scientificName, eventDate columns.
## This summary is simply the sum of complete.cases in each column. It ranges from zero to the N of columns. This will be used to sort duplicate rows and select the most-complete rows.
##  - Updating the .summary column to sort by...
##  - We will NOT flag the following columns. However, they will remain in the data file.
## .gridSummary, .lonFlag, .latFlag, .uncer_terms, .uncertaintyThreshold, .unLicensed
##  - summaryFun:
## Flagged 99 
##   The .summary column was added to the database.
##  - Working on CustomComparisonsRAW duplicates...
## 
## Completed iteration 1 of 1:
##  - Identified 0 duplicate records and kept 0 unique records using the column(s): 
## catalogNumber, institutionCode, scientificName
##  - Working on CustomComparisons duplicates...
## 
## Completed iteration 1 of 4:
##  - Identified 0 duplicate records and kept 0 unique records using the column(s): 
## gbifID, scientificName
## 
## Completed iteration 2 of 4:
##  - Identified 0 duplicate records and kept 0 unique records using the column(s): 
## occurrenceID, scientificName
## 
## Completed iteration 3 of 4:
##  - Identified 0 duplicate records and kept 0 unique records using the column(s): 
## recordId, scientificName
## 
## Completed iteration 4 of 4:
##  - Identified 0 duplicate records and kept 0 unique records using the column(s): 
## id, scientificName
##  - Working on collectionInfo duplicates...
## 
## Completed iteration 1 of 2:
##  - Identified 0 duplicate records and kept 0 unique records using the columns: 
## decimalLatitude, decimalLongitude, scientificName, eventDate, recordedBy, and catalogNumber
## 
## Completed iteration 2 of 2:
##  - Identified 0 duplicate records and kept 0 unique records using the columns: 
## decimalLatitude, decimalLongitude, scientificName, eventDate, recordedBy, and otherCatalogNumbers
##  - Clustering duplicate pairs...
## Duplicate pairs clustered. There are 0 duplicates across 0 kept duplicates.
##  - Ordering prefixs...
##  - Ordering data by 1. dataSource, 2. completeness and 3. .summary column...
##  - Find and FIRST duplicate to keep and assign other associated duplicates to that one (i.e., across multiple tests a 'kept duplicate', could otherwise be removed)...
##  - Duplicates have been saved in the file and location: /tmp/RtmpIYpfHt/Data_acquisition_workflow/Output/ReportduplicateRun_collectionInfo_2024-11-05.csv
##  - Across the entire dataset, there are now 0 duplicates from a total of 205 occurrences.
##  - Completed in 0.25 secs

check_time %>%
    readr::write_excel_csv(., paste(OutPath_Intermediate, "04_2_dup_database.csv",
        sep = "/"))

BeeBDC::flagRecorder(data = check_time, outPath = paste(OutPath_Report, sep = ""),
    fileName = paste0("flagsRecorded_", Sys.Date(), ".csv"), idColumns = c("database_id",
        "id", "catalogNumber", "occurrenceID", "dataSource"), append = TRUE, printSummary = TRUE)

if (!exists("duplicates")) {
    duplicates <- BeeBDC::fileFinder(path = DataPath, fileName = "duplicateRun_") %>%
        readr::read_csv()
}
##  - Dates found in file name(s). Finding most-recent file from file name...
##  - Found the following file(s): 
##  /tmp/RtmpIYpfHt/Data_acquisition_workflow/Output/Report/duplicateRun_collectionInfo_2024-11-05.csv
## Rows: 0 Columns: 19
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (19): group, database_id, database_id_keep, dupColumn_s, decimalLatitude...
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

# Make sure that the .summary column is updated
check_time <- BeeBDC::summaryFun(data = check_time, dontFilterThese = c(".gridSummary",
    ".lonFlag", ".latFlag", ".uncer_terms", ".uncertaintyThreshold"), removeFilterColumns = FALSE,
    filterClean = FALSE)
##  - We will NOT flag the following columns. However, they will remain in the data file.
## .gridSummary, .lonFlag, .latFlag, .uncer_terms, .uncertaintyThreshold
##  - summaryFun:
## Flagged 99 
##   The .summary column was added to the database.

# Save the uncleaned dataset
check_time %>%
    readr::write_excel_csv(., paste(OutPath_Intermediate, "05_unCleaned_database.csv",
        sep = "/"))

cleanData <- BeeBDC::summaryFun(
  data = check_time,
  dontFilterThese = c(".gridSummary", ".lonFlag", ".latFlag", ".uncer_terms",
                      ".uncertaintyThreshold"),
  # Remove the filtering columns?
  removeFilterColumns = TRUE,
  # Filter to ONLY cleaned data?
  filterClean = TRUE)
##  - We will NOT flag the following columns. However, they will remain in the data file.
## .gridSummary, .lonFlag, .latFlag, .uncer_terms, .uncertaintyThreshold
##  - summaryFun:
## Flagged 99 
##   The .summary column was added to the database.
##  - REMOVED all occurrences that were FALSE for the 'summary' column.

# Save this CLEANED dataset
cleanData %>%
    readr::write_excel_csv(., paste(OutPath_Intermediate, "05_cleaned_database.csv",
        sep = "/"))

if (!require("BiocManager", quietly = TRUE)) install.packages("BiocManager", repos = "http://cran.us.r-project.org")
BiocManager::install("ComplexHeatmap")

if (!exists("duplicates")) {
    duplicates <- BeeBDC::fileFinder(path = DataPath, fileName = "duplicateRun_") %>%
        readr::read_csv()
}

par(mar = c(2, 2, 2, 2)/2, mfrow = c(1, 1))

BeeBDC::chordDiagramR(
  # The duplicate data from the dupeSummary function output  
  dupeData = duplicates,
  outPath = OutPath_Figures,
  fileName = "ChordDiagram.pdf",
  # These can be modified to help fit the final pdf that's exported.
  width = 9,
  height = 7.5,
  bg = "white",
  # How few distinct dataSources should a group have to be listed as "other"
  smallGrpThreshold = 3,
  title = "Duplicated record sources",
  # The default list of colour palettes to choose from usign the paleteer package
  palettes = c("cartography::blue.pal", "cartography::green.pal", 
               "cartography::sand.pal", "cartography::orange.pal", "cartography::red.pal",
               "cartography::purple.pal", "cartography::brown.pal"),
  canvas.ylim = c(-1.0,1.0), 
  canvas.xlim = c(-0.6, 0.25),
  text.col = "black",
  legendX = grid::unit(6, "mm"),
  legendY = grid::unit(18, "mm"),
  legendJustify = c("left", "bottom"),
  niceFacing = TRUE)

if (!exists("check_time")) {
    beeData <- readr::read_csv(paste(OutPath_Intermediate, "05_unCleaned_database.csv",
        sep = "/"), col_types = BeeBDC::ColTypeR())
} else {
    beeData <- check_time
    rm(check_time)
}

BeeBDC::dupePlotR(
  data = beeData,
  # The outPath to save the plot as
  outPath = OutPath_Figures,
  fileName = "duplicatePlot.pdf",
  # Colours in order: duplicate, kept duplicate, unique
  dupeColours = c("#F2D2A2","#B9D6BC", "#349B90"),
  # Plot size and height
  base_height = 7, base_width = 7,
  legend.position = c(0.85, 0.8),
  # Extra variables can be fed into forcats::fct_recode() to change names on plot
  GBIF = "GBIF", SCAN = "SCAN", iDigBio = "iDigBio", USGS = "USGS", ALA = "ALA", 
  ASP = "ASP", CAES = "CAES", Ecd = "Ecd",
  returnPlot = TRUE
)

BeeBDC::plotFlagSummary(
  data = beeData,
  # Colours in order of pass (TRUE), fail (FALSE), and NA
  flagColours = c("#127852", "#A7002D", "#BDBABB"),
  fileName = paste0("FlagsPlot_", Sys.Date(),".pdf"),
  outPath = paste0(OutPath_Figures),
  width = 15, height = 9,
    # OPTIONAL:
      #   # Filter to a single species
      #       speciesName = "Holcopasites heliopsis",
      #         # column to look in
      #       nameColumn = "species",
      #        # Save the filtered data
      #       saveFiltered = TRUE,
      #   # Filter column to display on map
      #       filterColumn = ".summary",
      #       plotMap = TRUE,
      #   # amount to jitter points if desired, e.g. 0.25 or NULL
      #       jitterValue = NULL,
      #        # Map opacity value for points between 0 and 1
      #   mapAlpha = 1,
      #        # If a user wants to output the table used to make the figure, change this to TRUE
      #   saveTable = FALSE,
  # Extra variables can be fed into forcats::fct_recode() to change names on plot
  GBIF = "GBIF", SCAN = "SCAN", iDigBio = "iDigBio", USGS = "USGS", ALA = "ALA", 
  ASP = "ASP", CAES = "CAES", 'BMont' = "BMont", 'BMin' = "BMin", Ecd = "Ecd",
  Gaiarsa = "Gai", EPEL = "EPEL", VicWam = "VicWam",
  returnPlot = TRUE
)
##  - Preparing data to plot...
##  - Building plot...

if (!exists("cleanData")) {
    cleanData <- readr::read_csv(paste(OutPath_Intermediate, "05_cleaned_database.csv",
        sep = "/"), col_types = BeeBDC::ColTypeR())
}

if (!exists("cleanData")) {
    cleanData <- readr::read_csv(paste(OutPath_Intermediate, "05_cleaned_database.csv",
        sep = "/"), col_types = BeeBDC::ColTypeR(), locale = readr::locale(encoding = "UTF-8"))
}

# Note, if outPath = NULL then no file will be saved
dataProvTable <- BeeBDC::dataProvTables(data = cleanData, runBeeDataChecks = TRUE,
    outPath = NULL, fileName = "dataProvTable.csv")

# Note, if outPath = NULL then no file will be saved
summaryTable <- BeeBDC::flagSummaryTable(data = beeData, column = "scientificName",
    outPath = NULL, fileName = "flagTable.csv")
##  - We will flag all columns starting with '.'
##  - summaryFun:
## Flagged 111 
##   The .summary column was added to the database.
## The percentages of species impacted by each flag in your analysis are as follows: 
##   .coordinates_empty = 24.05%
##   .coordinates_outOfRange = 0%
##   .basisOfRecords_notStandard = 1.27%
##   .coordinates_country_inconsistent = 1.27%
##   .occurrenceAbsent = 8.86%
##   .unLicensed = 0%
##   .GBIFflags = 0%
##   .rou = 30.38%
##   .sequential = 0%
##   .uncertaintyThreshold = 15.19%
##   .eventDate_empty = 20.25%
##   .year_outOfRange = 16.46%
##   .duplicates = 0%