Updated / printed: 2023-09-07


Introduction

This script continues the data preparation methods needed for the Condition of Benthic Communities indicator (BH2b) in the UK EEZ Celtics Seas areas. Please see Data Preparation for previous data preparation steps.

This script focuses on;

  • filtering and cleaning the the data to the appropriate
    • mesh sizes
    • habitats
    • abundance record type
  • removing qualifiers and duplicate records
  • checking for any duplication of data between datasets

Data Preparation Part 2

## Load in the OneBenthic, Marine Recorder and WFD data from the previous script
# OneBenthic
OB_data_hab_cs <- readRDS("Data/Processed/R_data/C20230816_OB_data_hab_cs_bht")
# Marine Recorder
MR_data_hab_cs <- readRDS("Data/Processed/R_data/C20230823_MR_data_hab_cs_bht")
# WFD
WFD_data_hab_cs <- readRDS("Data/Processed/R_data/C20230526_WFD_data_hab_cs_bht")
# MEREMAN
merman_data_hab_cs <- readRDS("Data/Processed/R_data/C20230816_merman_data_hab_cs_bht")
# NRW 
samples_NRW_bht <- readRDS("Data/Processed/R_data/C20230831_samples_NRW_bht")

Mesh Size

Sample diversity will vary with the sieve mesh size used for the analysis, for comparable results (reference values) all samples within an assessment group (i.e., grab-core) should have the same mesh size.

In the CEMP guidelines (‘3.2 Quantitative Objectives’) sampling categories are suggested, for Grab-Core these are given as:

  • Sampled surface area around 0,1 m2 and mesh size of 0,1 cm

In previous drafts of data preparation both 0.5 mm and 1 mm mesh sizes samples were retained in the dataset. This was with the intention that 0.5 mm mesh size samples could potentially be assessed in addition to 1mm samples. There currently isn’t scope to include 0.5 mm mesh size samples at this time so they will be removed from the dataset

OneBenthic

  • Check what mesh sizes are included in the OneBenthic data
# Check the sample mesh size 
OB_data_hab_cs %>%
  as.data.frame() %>%
  select(samplecode, gearname, macrosievesizemm) %>%
  group_by(samplecode, gearname, macrosievesizemm) %>%
  dplyr::summarise(count = n_distinct(samplecode)) %>%
  select(gearname, macrosievesizemm, count) %>%
  group_by(gearname, macrosievesizemm) %>%
  dplyr::summarise(total = sum(count), .groups = "drop") %>%
    datatable(
        extensions = "Buttons",
        escape = FALSE,
        options = list(dom = "Blfrtip", buttons = c("csv")))

There are some unusual mesh sizes (>1)

  • Check records with mesh size >1
# Mesh >1 
OB_data_hab_cs %>%
  as.data.frame() %>%
  select(surveyname, gearname, macrosievesizemm, validnameworms, abund) %>%
  filter(macrosievesizemm >1)

Records with unusual mesh sizes do not have available data (data witheld)

  • retain only samples with 1 mm mesh size
OB_data_hab_cs_m <- OB_data_hab_cs %>%
  filter(macrosievesizemm == 1)

After filtering mesh size;

  • 1936 1mm samples

Marine Recorder

  • Check what mesh sizes are included in the Marine Recorder data
# Check the sample mesh size 
MR_data_hab_cs %>%
  as.data.frame() %>%
  select(samplekey, geartype, meshsize) %>%
  group_by(samplekey, geartype, meshsize) %>%
  dplyr::summarise(count = n_distinct(samplekey)) %>%
  select(geartype, meshsize, count) %>%
  group_by(geartype, meshsize) %>%
  dplyr::summarise(total = sum(count), .groups = "drop") %>%
    datatable(
        extensions = "Buttons",
        escape = FALSE,
        options = list(dom = "Blfrtip", buttons = c("csv")))

Frustratingly there are many samples without mesh size recorded, Day grabs in particular have 1587 samples without mesh size (1153 0.5mm and 1344 1mm).

  • Map sample points without analysis mesh information
# Load UKMS assessment units 
ukms <- st_read("Data/Raw/UKMS/C20230620_UKMS_final.shp")
# fix geometry
ukms <- st_make_valid(ukms)
# filter to Celtic Seas
ukms_cs <- filter(ukms, UKMS_SubRe == "Celtic Seas")
# Read in world landmass shapefile for creating maps
land <- st_read(paste0(NEA_PANACEA_folder, "Benthic Habitat AUs and Land Shapefiles/Benthic Habitat AUs and Land Shapefiles/c20210824_World_Outline_WGS84_LAEA.shp"))
# transform to WGS84
land <- st_transform(land, 4326)
# fix geometry 
land <- st_make_valid(land)

# clip the land to Celtic Seas area
sf::sf_use_s2(FALSE)
land_cs <- st_crop(land, ukms_cs)
sf::sf_use_s2(TRUE)
# Filter to samples without mesh size info, group to sample and drop unnecessary fields
MR_data_na_mesh <- MR_data_hab_cs %>%
  filter(is.na(meshsize)) %>%
  group_by(surveykey, samplekey, sampledate, sampleyear, geartype, msfd, eunis_l3, hab_type) %>%
  dplyr::summarise(N = sum(spcount, na.rm = TRUE), S = n_distinct(preferredspeciesname), .groups = "drop")
  
# Plot samples 
MR_data_na_mesh %>%
  ggplot() +
    geom_sf(data = land_cs, fill = "grey", colour = NA) +
    geom_sf(data = ukms_cs, mapping = aes(geometry = geometry), lwd = 0.5, colour = "grey40", fill = NA) +
    geom_sf(mapping = aes(geometry = geometry), colour = "blue", size = 1.2) +
    ylim(49.9, 60) +
    xlim(-9, -1) +
    theme_bw() +
    theme(legend.position = "none") +
    ggtitle("Marine Recorder Samples Without Mesh Size")

  • Summarise by survey name
# load in MR survey names
surveyname <- read.csv("Data/Processed/Marine_Recorder/SurveyName.csv")
# fix field name
surveyname <- surveyname %>%
  rename("SurveyKey" = "ï..SurveyKey") %>%
  mutate(SurveyKey = as.character(SurveyKey),
         SurveyName = as.character(SurveyName))
# add survey names to MR samples
MR_data_na_mesh_surveys <- left_join(MR_data_na_mesh, surveyname, by = c("surveykey" = "SurveyKey"))

# summarise surveys with NA mesh
MR_data_na_mesh_surveys %>%
  as.data.frame() %>%
  group_by(SurveyName, surveykey, geartype) %>%
  dplyr::summarise(sample_n = n()) %>%
    datatable(
        caption = "MR survey data with NA mesh size",
        colnames = c("samples" = "sample_n"),
        extensions = "Buttons",
        escape = FALSE,
        rownames = FALSE,
        options = list(dom = "Blfrtip", buttons = c("csv")))

Update missing mesh size information

The summary of survey without analysis mesh size metadata was shared with the Benthic Sub-Group (BSG) to fill in data gaps. Additionally, survey details available on the MEDIN portal were interrogated for evidence of the analysis methodology.

The following surveys were identified as having samples processed with a 1 mm sieve mesh:

# list of survey to be considered for analysis (1 mm)
surveys_1mm <- c("2010 SNH survey to establish the distribution of Priority Marine Features in the Clyde Sea area",
                 "2011 INIS Hydro Grab Survey",
                 "2011 Marine Scotland North Minch benthic infaunal survey",
                 "2011 Marine Scotland Small Isles benthic infaunal survey",
                 "2011 Marine Scotland survey of Lochs Linnhe, Etive, Leven and Eil: a broadscale validation - part A",
                 "2012 Belfast Ecosystem Assessment",
                 "2012 CEFAS Mounts Bay Video and Grab Survey",
                 "2012 Defra The Manacles rMCZ Site Verification Survey",
                 "2012-2013 AFBI Essential Fish Habitat Surveys",
                 "2013 Natural England Morecambe and Lune (Walney Channel) Benthic Grab Survey",
                 "2014 AFBI Fair Head Tidal Energy Development Site Survey",
                 "2014 AFBI Rathlin Deeps",
                 "2014-2015 AFBI Irish Sea Mud Grab Survey",
                 "2014-2016 AFBI Outer Ards Modiolus Surveys",
                 "2017 AFBI North Channel Habitat Mapping Project Benthic Infauna")

# Summary of surveys ad samples (with count data)
MR_data_na_mesh_surveys %>%
  as.data.frame() %>%
  filter(SurveyName %in% surveys_1mm, N > 0) %>%
  group_by(SurveyName) %>%
  tally()
  • Take list of all samples to be updated to 1.0 mm mesh size
    • Filter to those with abundance records (N > 0)
  • Update sample records to 1.0 meshsize
# take a list of sample keys for update
samplekey_1mm <- MR_data_na_mesh_surveys %>%
  as.data.frame() %>%
  filter(SurveyName %in% surveys_1mm, N > 0) %>%
  pull(samplekey)

# update records to 1 mm
MR_data_hab_cs <- MR_data_hab_cs %>%
  mutate(meshsize = if_else(samplekey %in% samplekey_1mm, 1.0, meshsize))

# tidy up 
rm(samplekey_1mm, MR_data_na_mesh_surveys, surveys_1mm)
  • Filter Marine Recorder data to 1mm mesh size
MR_data_hab_cs_m <- MR_data_hab_cs %>%
  filter(meshsize == 1)

After filtering mesh size;

  • 3474 1mm samples

EA WFD

  • Check which mesh sizes are included in the EA WFD data
# Check the sample mesh size 
WFD_data_hab_cs %>%
  as.data.frame() %>%
  select(samplecode, samplemethod, sievemeshum) %>%
  group_by(samplecode, samplemethod, sievemeshum) %>%
  dplyr::summarise(count = n_distinct(samplecode)) %>%
  select(samplemethod, sievemeshum, count) %>%
  group_by(samplemethod, sievemeshum) %>%
  dplyr::summarise(total = sum(count), .groups = "drop") %>%
    datatable(
        extensions = "Buttons",
        escape = FALSE,
        options = list(dom = "Blfrtip", buttons = c("csv")))
  • Filter WFD samples to 1 mm mesh size
WFD_data_hab_cs_m <- WFD_data_hab_cs %>%
  filter(sievemeshum == 1000)

After filtering mesh size;

  • 68 1mm samples

  • NRW and MERMAN data is already filtered to 1 mm mesh samples

Broad-Scale Habitats

The datasets still include samples with either no habitat information or not enough information to translate to broad-scale habitat definitions.

  • Filter to remove:
    • No habitat data
    • No EUNIS to MSFD BBHT translation

  • Filter Marine Recorder samples
    • 294 samples without sufficient habitat data
# filter MR
MR_data_hab_cs_m <- MR_data_hab_cs_m %>%
  filter(msfd != "No habitat data", msfd != "No EUNIS to MSFD BBHT translation")
  • Summary of broad-scale habitats and number of samples
# summary of BHTs
MR_data_hab_cs_m %>%
  as.data.frame() %>%
  group_by(samplekey, msfd) %>%
  summarise() %>%
  group_by(msfd) %>%
  count()

After filtering;

  • 3180 samples

  • Filter OneBenthic samples
# filter OB
OB_data_hab_cs_m <- OB_data_hab_cs_m %>%
  filter(msfd != "No habitat data", msfd != "No EUNIS to MSFD BBHT translation")
  • Summary of broad-scale habitats and number of samples
# summary of BHTs
OB_data_hab_cs_m %>%
  as.data.frame() %>%
  group_by(samplecode, msfd) %>%
  summarise() %>%
  group_by(msfd) %>%
  count()

After filtering:

  • 1726 sample

  • Filter WFD samples
# filter
WFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
  filter(msfd != "No habitat data", msfd != "No EUNIS to MSFD BBHT translation")
  • Summary of habitats and numbers of samples
# summary of BHTs
WFD_data_hab_cs_m %>%
  as.data.frame() %>%
  group_by(samplecode, msfd) %>%
  summarise() %>%
  group_by(msfd) %>%
  count()

After filtering:

  • 66 sample

  • Filter NRW samples
# filter
samples_NRW_bht <- samples_NRW_bht %>%
  filter(msfd != "No habitat data", msfd != "No EUNIS to MSFD BBHT translation")
  • Summary of habitats and numbers of samples
# summary of BHTs
samples_NRW_bht %>%
  as.data.frame() %>%
  group_by(msfd) %>%
  count()

After filtering:

  • 156 sample

  • Filter MERMAN samples
# filter
merman_data_hab_cs <- merman_data_hab_cs %>%
  filter(msfd != "No habitat data", msfd != "No EUNIS to MSFD BBHT translation")
  • Summary of habitats and numbers of samples
# summary of BHTs
merman_data_hab_cs %>%
  as.data.frame() %>%
  group_by(sampleref, msfd) %>%
  summarise() %>%
  group_by(msfd) %>%
  count()

After filtering:

  • 206 sample

A number of samples from each dataset are assigned rock and biogenic reef broad-scale habitats, either from sample information (Marine Recorder samples) or from spatially joining to the habitat map. As sampling has been preformed by means of grabs or cores the assignment of hard substrata is inappropriate, following the same approach to the QSR 2023 assessment these sample will be assigned to mixed sediment.

We know that rock and biogenic reef can’t be present in the data, as sampling has been performed by means of grabs, cores, dredges, and trawls. We will therefore rename “rock and biogenic reef” to “mixed sediment” (e-mail Sander 2022-07-18T12:13).

step-2-eda.html (BH2b in the Greater North Sea script)

Note, with this approach there are some samples with biogenic habitat metadata (e.g. A5.6 Sublittoral biogenic reefs) assigned to the mixed sediment broad-scale habitat groups.

  • Samples from Sublittoral biogenic reefs present in the Marine Recorder data
MR_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(grepl("A5.6", hab_type)) %>%
  group_by(samplekey, hab_type, msfd) %>%
  count() %>%
  group_by(hab_type, msfd) %>%
  count()
  • Update rock and biogenic reef to mixed sediment
MR_data_hab_cs_m <- MR_data_hab_cs_m %>%
  mutate(msfd = str_replace(msfd, "rock and biogenic reef", "mixed sediment"))

  • Samples from Sublittoral biogenic reefs present in the OneBenthic data
OB_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(grepl("A5.6", hab_type)) %>%
  group_by(samplecode, hab_type, msfd) %>%
  count() %>%
  group_by(hab_type, msfd) %>%
  count()
  • Update rock and biogenic reef to mixed sediment
OB_data_hab_cs_m <- OB_data_hab_cs_m %>%
  mutate(msfd = str_replace(msfd, "rock and biogenic reef", "mixed sediment"))

  • There are no samples from Sublittoral biogenic reefs present in the WFD data
  • Update rock and biogenic reef to mixed sediment
WFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
  mutate(msfd = str_replace(msfd, "rock and biogenic reef", "mixed sediment"))

  • Samples from Sublittoral biogenic reefs present in the OneBenthic data
merman_data_hab_cs %>%
  as.data.frame() %>%
  filter(grepl("A5.6", hab_type)) %>%
  group_by(sampleref, hab_type, msfd) %>%
  count() %>%
  group_by(hab_type, msfd) %>%
  count()
  • Update rock and biogenic reef to mixed sediment
merman_data_hab_cs <- merman_data_hab_cs %>%
  mutate(msfd = str_replace(msfd, "rock and biogenic reef", "mixed sediment"))

  • Samples from Sublittoral biogenic reefs present in the NRW data
samples_NRW_bht %>%
  as.data.frame() %>%
  filter(grepl("A5.6", hab_type)) %>%
  group_by(sampleref, hab_type, msfd) %>%
  count() %>%
  group_by(hab_type, msfd) %>%
  count()
  • Update rock and biogenic reef to mixed sediment
samples_NRW_bht <- samples_NRW_bht %>%
  mutate(msfd = str_replace(msfd, "rock and biogenic reef", "mixed sediment"))
  • Save filtered NRW data for analysis in the final script
# Processed NRW data
saveRDS(samples_NRW_bht, paste0("Data/Processed/R_data/", creation_date(), "_samples_NRW_bht"))

Clean and Standardise Datasets

The datasets need field names and data types standardising (count abundance) and any erroneous or duplicated data removing. Taxa and qualifier records also needed amalgamating to taxa level (removing qualifiers).

Common fields and equivalents between datasets:

Marine Recorder OneBenthic EA WFD MERMAN Description
samplekey samplecode samplecode sampleref sample ID
meshsize macrosievesizemm sievemeshum meshsize mesh size mm/um
sampledate sampledate sampledate samplestartdate raw sample date
sampleyear year sampleyear monitoringyear year sampled
speciesname validnameworms taxonname specieslatinname taxa name
preferredspeciesname validnameworms preferredtaxonname specieslatinname WoRMS taxa name / NHM name
preferredaphiaid aphiaid taxonlistitemkey NA WoRMS APHIA ID / NHM ID
spqual taxonqualifier qualifier stageofdevelopment/sexofspecies taxa qualifier
sacforn NA NA NA SACFOR abundance
sppercent NA NA NA percentage abundance
pa abund abundance abundance presents / absence
spcount abund abundance results abundance count
eunis NA NA NA sample EUNIS code (Marine Recorder metadata)
hab_type hab_type hab_type hab_type EUNIS from combined map
msfd msfd msfd msfd broad-scale habitat definitions

Marine Recorder

Duplicated Rows

  • Check for duplicates across all fields
# find all rows with duplicates in every field
dups <- MR_data_hab_cs_m %>%
  as.data.frame() %>%
  duplicated()
# list of duplicates
MR_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(dups) %>%
  select(samplekey, geartype, speciesname, spcount) %>%
  arrange(samplekey, speciesname)
  • 112 identical rows, these can all be removed (they’re exact copies)
    • Filter out exact duplicates
# filter duplicates
MR_data_hab_cs_m <- MR_data_hab_cs_m %>%
  filter(dups == FALSE)

# tidy up
rm(dups)

Present/Absent and SACFOR Abundance

The Marine Recorder data includes different measures of occurrence

  • Check some of the alternative recording categories for MR data (SACFOR, PA, %, No per sqm)
MR_data_hab_cs_m %>%
  as.data.frame() %>%
  select(sacforn) %>%
  group_by(sacforn) %>%
  dplyr::summarise(n = n(), .groups = "drop")
MR_data_hab_cs_m %>%
  as.data.frame() %>%
  select(pa) %>%
  group_by(pa) %>%
  dplyr::summarise(n = n(), .groups = "drop")
MR_data_hab_cs_m %>%
  as.data.frame() %>%
  select(sppercent) %>%
  group_by(sppercent) %>%
  dplyr::summarise(n = n(), .groups = "drop")
MR_data_hab_cs_m %>%
  as.data.frame() %>%
  select(numpersqm) %>%
  group_by(numpersqm) %>%
  dplyr::summarise(n = n(), .groups = "drop")
  • Check if number per sqm and spcount occur on independently
MR_data_hab_cs_m %>%
  as.data.frame() %>%
  select(spcount, numpersqm) %>%
  filter(is.na(spcount) & !is.na(numpersqm))
  • Remove sppercent field
# remove species percentage field
MR_data_hab_cs_m <- MR_data_hab_cs_m %>%
  select(-c(sppercent))
  • There are some rows with empty space ” ” in the pa field
    • Remove any empty space from the PA field
  • numpersqm is always present with spcount, preference to use spcount
    • Remove numpersqm
# trim and remove per sq m
MR_data_hab_cs_m <- MR_data_hab_cs_m %>%
  mutate(pa = str_trim(pa)) %>%
  select(-c(numpersqm))
  • Check for records without any abundance category (SACFOR, PA, or count)
# group by record type
MR_data_hab_cs_m %>%
  as.data.frame() %>%
  select(sacforn, pa, spcount) %>%
  mutate(count_included = if_else(is.na(spcount),"no","yes")) %>%
  group_by(sacforn, pa, count_included) %>%
  dplyr::summarise(n = n(), .groups = "drop")
  • SACFOR and PA records don’t have a count (will update these to an abundance of 1 later, as per CEMP guidelines)
    • The P/A field also includes “P” and “p”, update to “P”
# convert p to P
MR_data_hab_cs_m <- MR_data_hab_cs_m %>%
  mutate(pa = if_else(pa == "p", "P", pa))
  • There are 2578 records without any measure of abundance.

Taxa/qualifiers without abundance:

MR_data_hab_cs_m %>%
  as.data.frame() %>%
  select(speciesname, spqual, spcount, sacforn, pa) %>%
  filter(is.na(spcount), sacforn == "" | is.na(sacforn), pa == "" | is.na(pa)) %>%
  group_by(speciesname, spqual) %>%
  dplyr::summarise(n = n(), .groups = "drop") %>%
  arrange(speciesname) %>%
    datatable(
        caption = "Taxa without abundance records",
        extensions = "Buttons",
        rownames = FALSE,
        escape = FALSE,
        options = list(dom = "Blfrtip", buttons = c("csv")))
  • The most common taxa missing abundance records are those typically recorded as “Present” (hydroids, bryozoans, sponges, ‘eggs’).
    • However, there are taxa that would typically have a count record

QC a random selection of records without abundance:

MR_data_hab_cs_m %>%
  as.data.frame() %>%
  select(samplekey, speciesname, spqual, sacforn, pa, spcount, msfd) %>%
  filter(is.na(spcount), sacforn == "" | is.na(sacforn), pa == "" | is.na(pa)) %>%
  slice_sample(n = 20) 
  • Check all records in a random sample that has a taxa without abundance (MRNE015400002F35)
MR_data_hab_cs_m %>%
  as.data.frame() %>%
  select(samplekey, geartype, sampledate, speciesname, spqual, sacforn, pa, spcount) %>%
  filter(samplekey == "MRNE015400002F35") %>%
  arrange(speciesname) %>%
    datatable(
        extensions = "Buttons",
        rownames = FALSE,
        escape = FALSE,
        options = list(dom = "Blfrtip", buttons = c("csv")))
  • In this example sample (MRNE015400002F35) there are many rows without abundance, however there are records with counts and “Present” records. This suggests not all missing values are “Present”. There are also instances of taxa at multiple levels of classification (e.g. Amathia lendigera & Amathia) without abundance, it’s unlikely these taxa would be split like this in the same sample.

  • Check if the missing values are from a specific survey

MR_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(is.na(spcount), sacforn == "" | is.na(sacforn), pa == "" | is.na(pa)) %>%
  group_by(surveyeventkey) %>%
  dplyr::summarise(samples = n_distinct(samplekey), .groups = "drop")
  • Only 1-2 samples per survey event key, not specific to survey event
    • Check by start of survey event code and number with missing abundance
    • Compare with total samples with that starting survey code
# total of all samples
all <- MR_data_hab_cs_m %>%
  as.data.frame() %>%
  mutate(survey_part = str_sub(surveyeventkey, 0, 8)) %>%
  group_by(survey_part, sampleyear) %>%
  dplyr::summarise(samples_total = n_distinct(samplekey), .groups = "drop")
# samples missing abundance values
filtered <- MR_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(is.na(spcount), sacforn == "" | is.na(sacforn), pa == "" | is.na(pa)) %>%
  mutate(survey_part = str_sub(surveyeventkey, 0, 8)) %>%
  group_by(survey_part, sampleyear) %>%
  dplyr::summarise(samples = n_distinct(samplekey), .groups = "drop")
# summary table 
missing_values <- left_join(filtered, all, by = c("survey_part", "sampleyear"))

missing_values %>%
  arrange(survey_part, sampleyear)
# tidy up
rm(all, filtered, missing_values)
  • missing abundance values are not specific to any surveys

Filter out the taxa records missing abundance and keep the remaining as legitimate records.

  • Take a list of sample with empty abundance for reference
  • Build a quick comparison table of the number of records in each sample that is empty or has an abundance value
  • Filter out taxa records without abundance
# for reference, take a list of samples with missing abundance records
MR_samples_abun_na <- MR_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(is.na(spcount), sacforn == "" | is.na(sacforn), pa == "" | is.na(pa)) %>%
  select(samplekey) %>%
  unique() %>%
  pull()
# take the list as a temp df for joining
qc <- MR_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(is.na(spcount), sacforn == "", pa == "") %>%
  select(samplekey) %>%
  unique()
# join on samples missing abundance
qc_temp <- left_join(qc, MR_data_hab_cs_m, by = c("samplekey" = "samplekey"), suffix=c("",".y"))
# summarise missing abundance
qc_na <- qc_temp %>%
  as.data.frame() %>%
  group_by(samplekey) %>%
  dplyr::summarise(count_null = n(), .groups = "drop") 
# summarise those with abundance 
qc_abun <- qc_temp %>%
  as.data.frame() %>%
  filter(!is.na(spcount)| sacforn != ""| pa != "") %>%
  group_by(samplekey) %>%
  dplyr::summarise(count_abun = n(), .groups = "drop")
# combine missing and those with abundance  
left_join(qc_na, qc_abun, by = c("samplekey" = "samplekey"), suffix=c("",".y")) %>%
    datatable(
        extensions = "Buttons",
        rownames = FALSE,
        escape = FALSE,
        options = list(dom = "Blfrtip", buttons = c("csv")))
# tidy up temp data
rm(MR_samples_abun_na, qc, qc_temp, qc_na, qc_abun)

# filter rows with NA in all abundance fields
MR_samples_abun_m <- MR_data_hab_cs_m %>%
  filter(!is.na(spcount) | !is.na(sacforn) | !is.na(pa))
# filter empty SACFOR or P/A rows (without null count)
MR_samples_abun_m <- MR_samples_abun_m %>%
  filter(!is.na(spcount) | sacforn != "" | pa != "")

  • Check non-count taxa records
    • The presence/absence field (pa) had both P and N values
    • Check N taxa records
MR_samples_abun_m %>%
  as.data.frame() %>%
  select(samplekey, sampleyear, meshsize, preferredspeciesname, spqual, pa) %>%
  filter(pa == "N") %>%
  arrange(preferredspeciesname)
  • No obvious reasons from the the taxa names or qualifiers why N is used. However, all records seem to come from one survey (samplekey starting MRSNH029000003)

first 100 rows of MRSNH029000003 data:

MR_samples_abun_m %>%
  as.data.frame() %>%
  select(samplekey, sampleyear, preferredspeciesname, spqual, pa, spcount, sacforn) %>%
  filter(grepl("MRSNH029000003", samplekey)) %>%
  arrange(preferredspeciesname) %>%
  arrange(samplekey) %>%
  slice_head(n=100)
  • Looking through this sample, potentially “N” is for fragments, for example there are records of ‘bamboo worms’ like Owenia fusiformis and Nicomache lumbricalis (which typically break into lots of fragments) with “N” as abundance (some samples have O. fusiformis with counts and “N”)
  • Other taxa with “N” would typically be recorded as “Present” (bryozoa, seaweed etc.). However, there are other records with “P” in the samples so it’s still not clear why some have “N”.
  • There are 4 samples with “N” values:
MR_samples_abun_m %>%
  as.data.frame() %>%
  select(samplekey, pa) %>%
  filter(pa == "N") %>%
  distinct(samplekey)
  • Remove the 53 taxa records with “N”
MR_samples_abun_m <- MR_samples_abun_m %>%
  filter(pa != "N")
  • Check the SACFOR data
MR_samples_abun_m %>%
  as.data.frame() %>%
  select(samplekey, geartype, sampleyear, meshsize, preferredspeciesname, spqual, sacforn, pa, spcount) %>%
  filter(!is.na(sacforn) & sacforn != "") %>%
  arrange(preferredspeciesname)
  • All SACFOR data from MRNE01080000761D sample
    • Check all data in MRNE01080000761D sample
MR_samples_abun_m %>%
  as.data.frame() %>%
  select(samplekey, geartype, sampleyear, meshsize, preferredspeciesname, spqual, sacforn, pa, spcount) %>%
  filter(samplekey == "MRNE01080000761D") %>%
  arrange(preferredspeciesname)
  • MRNE01080000761D sample is all SACFOR or P/A
    • Filter MRNE01080000761D sample from dataset (not suitable for Margalef diversity)
    • Remove the redundant sacforn field
MR_samples_abun_m <- MR_samples_abun_m %>%
  filter(samplekey != "MRNE01080000761D") %>%
  select(-c(sacforn))

Numeric Count Abundance Records

  • Check if the spcount field includes fractions/decimal places (e.g. if samples have been sub-sampled and not rounded to integers)
MR_samples_abun_m %>%
  as.data.frame() %>%
  filter(!spcount%%1==0)
  • check the distribution of count records in the spcount field for any unusual frequencies
MR_samples_abun_m %>%
  as.data.frame() %>%
  filter(spcount <11) %>%
    ggplot() +
        geom_histogram(
            mapping = aes(spcount),
            colour = "black",
            breaks = seq(from = 0, to = 10, by = 1)) +
    scale_y_continuous(name = "frequency") +
    scale_x_continuous(breaks = c(1:10))

MR_samples_abun_m %>%
  as.data.frame() %>%
  filter(spcount >10 & spcount <101) %>%
    ggplot() +
        geom_histogram(
            mapping = aes(spcount),
            colour = "black",
            breaks = seq(from = 10, to = 100, by = 1)) +
    scale_y_continuous(name = "frequency") +
    scale_x_continuous(breaks = seq(10, 100, 10))

MR_samples_abun_m %>%
  as.data.frame() %>%
  filter(spcount >100 & spcount <1001) %>%
    ggplot() +
        geom_histogram(
            mapping = aes(spcount),
            colour = "black",
            breaks = seq(from = 100, to = 1000, by = 1)) +
    scale_y_continuous(name = "frequency") +
    scale_x_continuous(breaks = seq(100, 1000, 100))

MR_samples_abun_m %>%
  as.data.frame() %>%
  filter(spcount >1000) %>%
    ggplot() +
        geom_histogram(
            mapping = aes(spcount),
            colour = "black",
            breaks = seq(from = 1000, to = 6000, by = 1)) +
    scale_y_continuous(name = "frequency", breaks = c(1, 2, 1))

  • As expected, frequency of abundance values rapidly declines with higher values
    • There is no obvious sign of anomalous peaks for any values
  • Check some of the higher abundance records
MR_samples_abun_m %>%
  as.data.frame() %>%
  select(samplekey, sampleyear, meshsize, preferredspeciesname, spqual, spcount) %>%
  filter(spcount >1000) %>%
  arrange(preferredspeciesname)
  • Although very high, these abundance values are plausible and don’t require removing as errors

Taxa Name

The dataset still contains two sets of taxa name (speciesname and preferredspeciename) and two Aphia ID fields (aphiid and preferredaphiaid). To avoid confusion and aid standardisation we will just keep the ‘preferred’ names and IDs.

  • First check for empty name fields
MR_samples_abun_m %>%
  as.data.frame() %>%
  select(speciesname, preferredspeciesname, aphiaid, preferredaphiaid) %>%
  filter(is.na(speciesname) | is.na(preferredspeciesname)) %>%
  group_by(speciesname) %>%
  arrange(speciesname)
  • No records missing ‘preferred’ names
  • Drop the speciesname and aphiaid fields and use preferrredspeciesname and preferredaphiaid as the taxa name and aphia fields
  • Also drop the uncertain field, this doesn’t have any consistency or use (and causes problems when grouping)
MR_samples_abun_m <- MR_samples_abun_m %>%
  select(-c(speciesname, aphiaid, uncertain))

Duplicate Taxa with Count and P/A Records

  • Check for duplicates of taxa within samples (inc. with different count or P/A values)
# find all rows with duplicates in every field except count or P/A
dups <- MR_samples_abun_m %>%
  as.data.frame() %>%
  select(-c(spcount, pa, geometry)) %>%
  duplicated()
  • 83 samples with duplicate taxa records (this includes qualifiers, there will be many duplicates without qualifiers)
    • These duplicates must have the same ‘taxa & qualifier’ recorded more than once in the same sample but with different counts or P/A’s
  • Check the records with these duplicates
MR_samples_abun_m %>%
  as.data.frame() %>%
  select(-c(pa, spcount, geometry)) %>%
  group_by_all() %>%
  tally() %>%
  filter(n > 1)
  • Check where duplicates are from species with a count and “Presence” record
# pull a list of duplicates taxa in the same sample
qc <- MR_samples_abun_m %>%
  as.data.frame() %>%
  select(c(samplekey, preferredspeciesname, spqual)) %>%
  group_by_all() %>%
  tally() %>%
  filter(n > 1)

# join on samples with duplicates to create a subset
qc_temp <- left_join(qc, MR_samples_abun_m, by = c("samplekey" = "samplekey", "preferredspeciesname" = "preferredspeciesname", "spqual" = "spqual"), suffix=c("",".y"))

# check values for the duplicates
qc_temp %>%
  select(samplekey, preferredspeciesname, spqual, pa, spcount) %>%
  arrange(preferredspeciesname, samplekey)
  • The “present” records are alongside counts (although L. viridis has a count when it should be a “P”, these will be converted later)
    • Remove these duplicate presence records to avoid double counting
qc_temp <- qc_temp %>%
  filter(pa != "P")
  • Sum those that remain (they have different counts, and therefore likely they were split by additional names/qualifiers not captured in MR)
qc_temp <- qc_temp %>%
  group_by(samplekey, preferredspeciesname, spqual) %>%
  dplyr::summarise(spcount_sum = sum(spcount), .groups = "drop")
  • Bring the subset back to the main dataset
    • update spcount values
    • Remove the duplicated ‘presence’ and count taxa
    • Remove joint sum field
    • Remove the now duplicated records with sum values
# join the summed abundance
MR_samples_abun_m_temp <- left_join(MR_samples_abun_m, qc_temp, by = c("samplekey" = "samplekey", "preferredspeciesname" = "preferredspeciesname", "spqual" = "spqual"), suffix=c("",".y"))

# update the count field for those with a sum
MR_samples_abun_m_temp <- MR_samples_abun_m_temp %>%
  mutate(spcount = if_else(is.na(spcount_sum), spcount, spcount_sum))

# remove the duplicated rows that had "present" row and count row
# remove the now redundant sum field 
MR_samples_abun_m_temp <- MR_samples_abun_m_temp %>%
  filter(is.na(pa) | pa == "" | is.na(spcount)) %>%
  select(-c(spcount_sum))

# remove duplicates (those with a sum count will now be duplicates)
MR_samples_abun_m <- MR_samples_abun_m_temp %>%
  distinct()

# tidy up the temp data
rm(qc, qc_temp, MR_samples_abun_m_temp)

Duplicate Taxa with Different Qualifiers

  • First check for records that can be removed based on quailier (e.g. records of ‘eggs’ or ‘dead’ specimens)
MR_samples_abun_m %>%
  as.data.frame() %>%
  select(spqual) %>%
  filter(spqual != "") %>%
  group_by(spqual) %>%
  tally() %>%
  arrange(spqual)
  • Many different qualifiers recorded
    • Remove records of dead, eggs, and the spionids recorded as tubes
MR_samples_abun_m %>%
  as.data.frame() %>%
  select(spqual) %>%
  filter(grepl("dead|egg|tubes", spqual, ignore.case = TRUE)) %>%
  group_by(spqual) %>%
  tally() %>%
  arrange(spqual)
MR_samples_abun_m <- MR_samples_abun_m %>%
  filter(grepl("dead|egg|tubes", spqual, ignore.case = TRUE) == FALSE)

With taxa names now (partially) standardised to MR ‘preferred names’ we can remove the qualifiers and combine the duplicates. This will aggregate any taxa recorded multiple times in the same sample (e.g. juveniles and adults or species types) and also amalgamate taxa split with different recording policies (e.g. “Syllis sp. A”, “Syllis sp. B”) to the same recorded taxa name (Syllis).

  • we will also convert “presence” records to a count of “1”
  • Remove qualifier and present/absent field
MR_samples_abun_m <- MR_samples_abun_m %>%
  mutate(spcount = if_else(pa == "P", 1, as.numeric(spcount)))

MR_samples_abun_m <- MR_samples_abun_m %>%
  select(-c(spqual, pa))
  • Group the data to sum counts of the same taxa in a sample
    • Note, geometry has to be dropped before summarising and then joined after
# pull a list of column names except for the count
fields <- colnames(MR_samples_abun_m)[colnames(MR_samples_abun_m) != 'spcount' & colnames(MR_samples_abun_m) != 'geometry']

MR_samples_count <- MR_samples_abun_m %>%
  as.data.frame() %>%
  group_by(across(all_of(fields))) %>%
  dplyr::summarise(count_n = sum(spcount)) %>%
  ungroup()
# Note - geometry needs re-joining 

# Take a data frame of the coordinates per sample
coordinates <- MR_samples_abun_m %>%
  select(samplekey, surveykey, geometry) %>%
  distinct()

# Join to the aggregated sample data
MR_samples_count <- left_join(MR_samples_count, coordinates, by = c("samplekey" = "samplekey", "surveykey" = "surveykey"))

# tidy up
rm(fields, coordinates)
  • 356 rows amalgamated (identical taxa names previously split by qualifiers)
  • Check there are no duplicates
MR_samples_count %>%
  as.data.frame() %>%
  group_by(samplekey, preferredspeciesname) %>%
  dplyr::summarise(n = n()) %>%
  filter(n>1)

Duplicate Samples (replicates)

  • Check for duplicate samplekey samples with different repid values
MR_samples_count %>%
  as.data.frame() %>%
  select(samplekey, repid) %>%
  group_by(samplekey,repid) %>%
  dplyr::summarise(n = n()) %>% 
  group_by(samplekey) %>%
  tally() %>%
  filter(n >1) %>%
  arrange(desc(n))
  • No duplicate samplekey samples with different repid (samplekey is unique)

  • Check for replicates by (exact) location and date

MR_samples_count %>%
  as.data.frame() %>%
  select(samplekey, sampledate, geometry, geartype) %>%
  mutate(coords = as.character(geometry)) %>%
  group_by(samplekey, sampledate, coords, geartype) %>%
  dplyr::summarise(n = n()) %>% 
  group_by(coords, sampledate, geartype) %>%
  tally() %>%
  filter(n >1) %>%
  arrange(desc(n))

As expected there are some samples that have been collected as replicates, although the total number is low. However, there are some erroneous sample locations highlighted by this aggregation ((-7.000006, 50))

  • Investigate the samples with (-7.000006, 50) coordinates
    • Samples are from Greater Haig Fras 2012 survey (MRCON04000000005)
# plot survey with these coords
MR_samples_count %>%
  filter(surveykey == "MRCON04000000005") %>%
  ggplot() +
  geom_sf(mapping = aes(geometry = geometry))

# samples from this survey
MR_samples_count %>%
  as.data.frame() %>%
  filter(surveykey == "MRCON04000000005") %>%
  mutate(coords = as.character(geometry)) %>%
  group_by(samplekey, coords) %>%
  count() %>%
  arrange(desc(coords))

The coordinates for these samples have an error, we will remove them from the dataset

# filter "(-7.000006, 50)" samples
MR_samples_count <- MR_samples_count %>%
  filter(!grepl("MRCON040000001D6|MRCON040000001DA|MRCON040000001F0|MRCON040000001F8|MRCON040000001FC|MRCON04000000206|MRCON0400000020C|MRCON04000000218", samplekey))

Total replicate samples:

MR_samples_count %>%
  as.data.frame() %>%
  select(samplekey, sampledate, geometry, geartype) %>%
  mutate(coords = as.character(geometry)) %>%
  group_by(samplekey, sampledate, coords, geartype) %>%
  dplyr::summarise(n = n()) %>% 
  group_by(coords, sampledate, geartype) %>%
  tally() %>%
  filter(n >1) %>%
  ungroup() %>%
  dplyr::summarise(total = sum(n))

For the assessment all samples (i.e., all replicates individually) will have Margalef calculated, where replicates are present the median of Margalef D will be taken for the final assessment.


Save processed data

  • Save the cleaned Marine Recorder data for the next script
# Processed Marine Recorder data
saveRDS(MR_samples_count, paste0("Data/Processed/R_data/", creation_date(), "_MR_samples_count"))

OneBenthic

Duplicated Rows

  • Check for duplicates across all fields
# find all rows with duplicates in every field
dups <- OB_data_hab_cs_m %>%
  as.data.frame() %>%
  duplicated()
# list of duplicates
OB_data_hab_cs_m  %>%
  as.data.frame() %>%
  filter(dups) %>%
  select(samplecode, year, gearname, validnameworms, abund)
  • Example of a sample with duplicates (ISLAYOWF2012_42)
# example of duplicates
OB_data_hab_cs_m  %>%
  as.data.frame() %>%
  select(samplecode:hab_type) %>%
  filter(samplecode == "ISLAYOWF2012_42") %>%
  arrange(validnameworms)
  • Only a few identical records (11), as these are all identical records within a sample we will remove them from the data.
    • Filter out exact duplicates
OB_data_hab_cs_m <- OB_data_hab_cs_m %>%
  filter(dups == FALSE)

# tidy up
rm(dups)

Abundance Values

  • Check abundance values
# group by abundance value and check
OB_data_hab_cs_m %>%
  as.data.frame() %>%
  select(abund) %>%
  group_by(abund) %>%
  dplyr::summarise(n = n(), .groups = "drop") %>%
  arrange(desc(n))
  • The abund field contains values with data witheld (396 records).
OB_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(abund == "data witheld") %>%
  group_by(surveyname, samplecode, year) %>%
  count() %>%
  group_by(surveyname, year) %>%
  summarise(samples = sum(n)) %>%
  arrange(surveyname)
## `summarise()` has grouped output by 'surveyname'. You can override using the `.groups` argument.
  • Check if surveys with abundance recorded as data witheld also contain normal records
# take a list of surveys with "data witheld"
surveys_witheld <- OB_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(abund == "data witheld") %>%
  group_by(surveyname) %>%
  count() %>%
  chuck("surveyname")
# check for different abundance records
OB_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(surveyname %in% surveys_witheld) %>%
  group_by(abund) %>%
  count()
# check taxa names
OB_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(surveyname %in% surveys_witheld) %>%
  group_by(validnameworms) %>%
  count()
# tidy up temp data
rm(surveys_witheld)
  • All surveys with data witheld abundance also have data with held for taxa names and no normal sample records.
  • Remove surveys where all sample data is witheld (i.e. abundance = data with held)
OB_data_hab_cs_m <- OB_data_hab_cs_m %>%
  filter(abund != "data witheld")

  • Now abund doesn’t include any characters it can be converted to number format
  • First check if the abund field includes fractions/decimal places (e.g. if samples have been sub-sampled and not rounded to integers)
OB_data_hab_cs_m %>%
  as.data.frame() %>%
  select(abund) %>%
  mutate(abund = as.double(abund))%>%
  filter(!abund%%1==0)
OB_data_hab_cs_m <- OB_data_hab_cs_m %>%
  mutate(abund = as.integer(abund))

Frequency of Count Records

  • check the distribution of count records in the spcount field
OB_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(abund <11) %>%
    ggplot() +
        geom_histogram(
            mapping = aes(abund),
            colour = "black",
            breaks = seq(from = 0, to = 10, by = 1)) +
    scale_y_continuous(name = "frequency") +
    scale_x_continuous(breaks = c(1:10))

OB_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(abund >10 & abund <101) %>%
    ggplot() +
        geom_histogram(
            mapping = aes(abund),
            colour = "black",
            breaks = seq(from = 10, to = 100, by = 1)) +
    scale_y_continuous(name = "frequency") +
    scale_x_continuous(breaks = seq(10, 100, 10))

OB_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(abund >100 & abund <1001) %>%
    ggplot() +
        geom_histogram(
            mapping = aes(abund),
            colour = "black",
            breaks = seq(from = 100, to = 1000, by = 1)) +
    scale_y_continuous(name = "frequency") +
    scale_x_continuous(breaks = seq(100, 1000, 100))

OB_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(abund >1000) %>%
    ggplot() +
        geom_histogram(
            mapping = aes(abund),
            colour = "black",
            breaks = seq(from = 1000, to = 3000, by = 1)) +
    scale_y_continuous(name = "frequency", breaks = c(1, 2, 1))

  • As expected, frequency of abundance values rapidly declines with higher values
    • There is no obvious sign of anomalous peaks for any values
  • Check some of the higher abundance records
OB_data_hab_cs_m %>%
  as.data.frame() %>%
  select(surveyname, year, macrosievesizemm, validnameworms, taxonqualifier, abund) %>%
  filter(abund >500) %>%
  arrange(validnameworms)

All high abundance records are taxa that are not unusual to occur in high numbers


Duplicate Taxa Records

  • Check for duplicates of taxa within samples (taxon & qualifier with different count values)
# find all rows with duplicates in every field except count or P/A
dups <- OB_data_hab_cs_m %>%
  as.data.frame() %>%
  select(-c(abund, geometry)) %>%
  duplicated()
# list of duplicates
OB_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(dups) %>%
  select(samplecode, year, gearname, validnameworms, taxonqualifier) %>%
  arrange(validnameworms, samplecode)

17 samples with duplicate taxa records (this includes qualifiers, there will be many more duplicates with qualifiers removed)

  • Sum taxa records with identical taxa name and qualifier in a sample (likely they were split by additional names/qualifiers not captured in the dataset)
qc_temp <- OB_data_hab_cs_m %>%
  as.data.frame() %>%
  group_by(samplecode, samplecode2, date, validnameworms, taxonqualifier) %>%
  dplyr::summarise(abund_sum = sum(abund), .groups = "drop")
  • Update any duplicates with the summed abundance values
# join the summed abundance
OB_samples_abun_temp <- left_join(OB_data_hab_cs_m, qc_temp, by = c("samplecode" = "samplecode", "samplecode2" = "samplecode2", "date" = "date",  "validnameworms" = "validnameworms", "taxonqualifier" = "taxonqualifier"), suffix=c("",".y"))

# update the count field for those with a sum, remove the sum field
OB_samples_abun_temp <- OB_samples_abun_temp %>%
  mutate(abund = if_else(is.na(abund_sum), abund, abund_sum)) %>%
  select(-c(abund_sum))

# remove duplicates (those with a sum count will now be duplicates)
OB_samples_abun <- OB_samples_abun_temp %>%
  distinct()

# tidy up the temp data
rm(OB_samples_abun_temp, dups)

Duplicate Taxa with Different Qualifiers

  • First check for records that can be removed based on quailier (e.g. records of ‘eggs’ or ‘dead’ specimens)
OB_samples_abun %>%
  as.data.frame() %>%
  select(taxonqualifier) %>%
  filter(taxonqualifier != "") %>%
  group_by(taxonqualifier) %>%
  tally() %>%
  arrange(taxonqualifier)

Qualifiers are relatively standardised, although there are many with data witheld

  • Check survey/sample with data witheld qualifier
OB_samples_abun %>%
  as.data.frame() %>%
  filter(taxonqualifier == "data witheld") %>%
  group_by(surveyname, samplecode) %>%
  summarise(records = n()) %>%
  group_by(surveyname) %>%
  tally()

135 samples with data witheld qualifier

  • QC check a random survey with data witheld qualifier (Croker Carbonate Slabs MPA 2015_MPACRKS2015)
OB_samples_abun %>% 
  as.data.frame() %>% 
  filter(surveyname == "Croker Carbonate Slabs MPA 2015_MPACRKS2015") %>%
  select(samplecode, validnameworms, taxonqualifier, abund) %>%
  arrange(validnameworms) %>%
  slice_sample(n =100)

There are regular qualifiers amongst these records (e.g. juvenile) but no “null” qualifiers, suggests data witheld is likely to represent NA values and we shouldn’t exclude samples that contain this qualifier.

  • Remove records of eggs and seeds
OB_samples_abun <- OB_samples_abun %>%
  filter(!grepl("eggs|seeds", taxonqualifier, ignore.case = TRUE))

With unwanted qualifier records removed (e.g. records of “eggs”) we can remove the qualifier field and aggregate any taxa recorded in the same sample with different qualifiers (e.g. juveniles and adults with varying recording policies).

  • Remove qualifier field
OB_samples_abun <- OB_samples_abun %>%
  select(-c(taxonqualifier))
  • Group the data to sum counts of the same taxa in a sample
    • Note, geometry has to be dropped before summarising and then joined after
# pull a list of column names except for the abundance
fields <- colnames(OB_samples_abun)[colnames(OB_samples_abun) != 'abund' & colnames(OB_samples_abun) != 'geometry']

OB_samples_count <- OB_samples_abun %>%
  as.data.frame() %>%
  group_by(across(all_of(fields))) %>%
  dplyr::summarise(count_n = sum(abund)) %>%
  ungroup()
# Note - geometry needs re-joining 

# Take a data frame of the coordinates per sample
coordinates <- OB_samples_abun %>%
  select(samplecode, surveyname, geometry) %>%
  distinct()

# Join to the aggregated sample data
OB_samples_count <- left_join(OB_samples_count, coordinates, by = c("samplecode" = "samplecode", "surveyname" = "surveyname"))

# tidy up
rm(fields, coordinates)

124 rows amalgamated (identical taxa names previously split by qualifiers)

  • Check there are no duplicates
OB_samples_count %>%
  as.data.frame() %>%
  group_by(samplecode, validnameworms) %>%
  dplyr::summarise(n = n()) %>%
  filter(n>1)

Duplicate Samples (replicates)

  • Check for duplicate samplecode samples with different samplecode2 values
OB_samples_count %>%
  as.data.frame() %>%
  select(samplecode, samplecode2) %>%
  group_by(samplecode, samplecode2) %>%
  dplyr::summarise(n = n()) %>% 
  group_by(samplecode) %>%
  tally() %>%
  filter(n >1) %>%
  arrange(desc(n))
  • No duplicate samplecode samples with different samplecode2 (samplecode is unique)

  • Check for replicates by location and date

OB_samples_count %>%
  as.data.frame() %>%
  select(surveyname, samplecode, date, geometry, gearname) %>%
  mutate(coords = as.character(geometry)) %>%
  group_by(surveyname, samplecode, date, coords, gearname) %>%
  dplyr::summarise(n = n()) %>% 
  group_by(coords, date, surveyname, gearname) %>%
  tally() %>%
  filter(n >1) %>%
  arrange(surveyname, coords, date)

Some instances of replicates, as to be expected.


Save Processed Data

Save the data for processing in the next script

# Processed OneBenthic data
saveRDS(OB_samples_count, paste0("Data/Processed/R_data/", creation_date(), "_OB_samples_count"))

EA WFD

Duplicated Rows

  • Check for duplicate records
# find any rows with duplicates in every field
dups <- WFD_data_hab_cs_m %>%
  as.data.frame() %>%
  duplicated()
# list of duplicates
WFD_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(dups)
# tidy up
rm(dups)
  • No identical records in the dataset

Abundnance Records

  • Check record types for abundance
# unique abundance values
WFD_data_hab_cs_m %>%
  as.data.frame() %>%
  select(abundance) %>%
  unique() %>%
  arrange(abundance)
# check there is no 'na' abundance
WFD_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(is.na(abundance))

All abundance records are integers or “P” for present

Numeric Count Abundance Records

check the distribution of count records in the abundance field for any unusual frequencies/values

WFD_data_hab_cs_m  %>%
  as.data.frame() %>%
  filter(abundance != "P") %>%
  mutate(abundance = as.numeric(abundance)) %>%
    ggplot() +
        geom_histogram(
            mapping = aes(abundance),
            colour = "black",
            breaks = seq(from = 0, to = 30, by = 1)) +
    scale_y_continuous(name = "frequency") +
    scale_x_continuous(breaks = c(1:30))

WFD_data_hab_cs_m  %>%
  as.data.frame() %>%
  filter(abundance != "P") %>%
  mutate(abundance = as.numeric(abundance)) %>%
    ggplot() +
        geom_histogram(
            mapping = aes(abundance),
            colour = "black",
            breaks = seq(from = 30, to =150, by = 1)) +
    scale_y_continuous(name = "frequency", breaks = seq(0, 16, 2)) +
    scale_x_continuous(breaks = seq(30, 150, 10))

WFD_data_hab_cs_m  %>%
  as.data.frame() %>%
  filter(abundance != "P") %>%
  mutate(abundance = as.numeric(abundance)) %>%
    ggplot() +
        geom_histogram(
            mapping = aes(abundance),
            colour = "black",
            breaks = seq(from = 150, to = 800, by = 1)) +
    scale_y_continuous(name = "frequency", breaks = seq(0, 3, 1)) +
    scale_x_continuous(breaks = seq(150, 800, 50))

As expected, frequency of abundance values rapidly declines with higher values

  • check the higher abundance values for any unsusual records
WFD_data_hab_cs_m  %>%
  as.data.frame() %>%
  filter(abundance != "P") %>%
  mutate(abundance = as.numeric(abundance)) %>%
  select(surveyname, samplecode, samplemethod, preferredtaxonname, qualifier, abundance) %>%
  filter(abundance > 250) %>%
  arrange(preferredtaxonname)

No unusual abundance records (Carrick Roads samples are notoriously abundant!)

Taxon Name

The EA WFD data includes recorded taxa name (taxonname) and NHM nomenclature preferred names (preferredtaxonname)

  • Check for differences between the names
WFD_data_hab_cs_m %>%
  as.data.frame() %>%
  select(taxonname, preferredtaxonname) %>%
  filter(taxonname != preferredtaxonname) %>%
  unique() %>%
  arrange(taxonname)
  • Out of the differences between the two the NHM preferredtaxonname are most up to date
    • Remove taxonname and keep preferredtaxonname for matching with WoRMS later
WFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
  select(-c(taxonname))

Duplicate Taxa Records

  • Check for duplicates in samples with different abundance records
# find any rows with duplicates in every field except count or geometry
dups <- WFD_data_hab_cs_m %>%
  as.data.frame() %>%
  select(-c(abundance, geometry)) %>%
  duplicated()
# list duplicates
WFD_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(dups) %>%
  select(samplecode, sampleyear, samplemethod, preferredtaxonname, qualifier) %>%
  arrange(preferredtaxonname, samplecode)

Only 2 duplicate records

  • Sum these records to avoid duplicate taxon records in a sample
    • As abundance still includes characters and there are only two records we will do this individually for each record
# update duplicate abundance and remove
WFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
  mutate(abundance = if_else(samplecode == "CRT19S001B" & preferredtaxonname == "Copepoda" & abundance == "27", "28", abundance)) %>%
  mutate(abundance = if_else(samplecode == "CRT19S003B" & preferredtaxonname == "Copepoda" & abundance == "16", "17", abundance)) %>%  
  filter(!dups)
# tidy up
rm(dups)

Duplicate Taxa with Different Qualifiers

First we will check what qualifiers are present

WFD_data_hab_cs_m %>%
  as.data.frame() %>%
  select(qualifier) %>%
  filter(qualifier != "") %>%
  group_by(qualifier) %>%
  tally() %>%
  arrange(qualifier)
  • Remove records of eggs
WFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
  filter(!grepl("eggs", qualifier, ignore.case = TRUE))

We can now remove the qualifier field and aggregate any taxon recorded in a sample with different qualifiers

  • Remove quailifer
WFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
  select(-c(qualifier))
  • Check for taxa recorded as present and with a count after quailifers removed
# find "P" records
WFD_presence <- WFD_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(abundance == "P") %>%
  select(samplecode, preferredtaxonname, abundance)
# find count records
WFD_count <- WFD_data_hab_cs_m %>%
  as.data.frame() %>%
  filter(abundance != "P") %>%
  select(samplecode, preferredtaxonname, abundance)
# find those that have both count and P records
WFD_presence_dup <- inner_join(WFD_presence, WFD_count, by = c("samplecode" = "samplecode", "preferredtaxonname" = "preferredtaxonname"), suffix = c("", ".y")) %>%
  print()
##   samplecode preferredtaxonname abundance abundance.y
## 1 CRT19S010B          Spionidae         P           2
## 2 CRT19S005B          Spionidae         P          56
## 3 CRT19S012B          Spionidae         P          22
  • Remove these duplicate presence records (keeping the count values)
# filter by join
WFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
  anti_join(WFD_presence_dup, by = c("samplecode" = "samplecode", "preferredtaxonname" = "preferredtaxonname", "abundance" = "abundance"))
# tidy up
rm(WFD_presence, WFD_count, WFD_presence_dup)

Now there are no duplicate presence and count records for the same taxa in a sample we can convert all “P” records to 1. At later stages taxonomic recording policy (presence/absence or count) will be further standradised in line with the BH2b CEMP guidelines.

  • Update P to 1
    • Convert abundance to a numeric field
WFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
  mutate(abundance = if_else(abundance == "P", "1", abundance),
         abundance = as.numeric(abundance))

Now all abundance values are numeric we can aggregate duplicate taxa in samples (previously with different quailifiers)

  • Group the data to sum counts of the same taxa in a sample
    • Note, geometry has to be dropped before summarising and then joined after
# pull a list of column names except for the abundance
fields <- colnames(WFD_data_hab_cs_m)[colnames(WFD_data_hab_cs_m) != 'abundance' & colnames(WFD_data_hab_cs_m) != 'geometry']

WFD_data_hab_cs_count <- WFD_data_hab_cs_m %>%
  as.data.frame() %>%
  group_by(across(all_of(fields))) %>%
  dplyr::summarise(count_n = sum(abundance)) %>%
  ungroup()
# Note - geometry needs re-joining 

# Take a data frame of the coordinates per sample
coordinates <- WFD_data_hab_cs_m  %>%
  select(samplecode, surveyname, geometry) %>%
  distinct()

# Join to the aggregated sample data
WFD_samples_count <- left_join(WFD_data_hab_cs_count, coordinates, by = c("samplecode" = "samplecode", "surveyname" = "surveyname"))

# tidy up
rm(fields, coordinates)

36 rows amalgamated (identical taxa names previously split by qualifiers)

  • Check there are no duplicates
WFD_samples_count %>%
  as.data.frame() %>%
  group_by(samplecode, preferredtaxonname) %>%
  dplyr::summarise(n = n()) %>%
  filter(n>1)

Duplicate Samples (replicates)

  • Check for replicates by location and date
WFD_samples_count %>%
  as.data.frame() %>%
  select(surveyname, samplecode, sampledate, geometry, samplemethod) %>%
  mutate(coords = as.character(geometry)) %>%
  group_by(surveyname, samplecode, sampledate, coords, samplemethod) %>%
  dplyr::summarise(n = n()) %>% 
  group_by(coords, sampledate, surveyname, samplemethod) %>%
  tally() %>%
  filter(n >1) %>%
  arrange(surveyname, coords, sampledate)

No duplicate samples taken with exactly the same location, date and gear type


Save Processed Data

Save the EA WFD data for processing in the next script

# Processed WFD data
saveRDS(WFD_samples_count, paste0("Data/Processed/R_data/", creation_date(), "_WFD_samples_count"))

MERMAN

Duplicated Rows

  • Check for duplicate records
# find any rows with duplicates in every field
dups <- merman_data_hab_cs %>%
  as.data.frame() %>%
  duplicated()
# list of duplicates
merman_data_hab_cs %>%
  as.data.frame() %>%
  filter(dups)
# tidy up
rm(dups)
  • No identical records in the dataset

Abundnance Records

  • Check record types for abundance and result
# unique abundance values
merman_data_hab_cs %>%
  as.data.frame() %>%
  select(abundance) %>%
  unique() %>%
  arrange(abundance)
merman_data_hab_cs %>%
  as.data.frame() %>%
  select(result) %>%
  unique() %>%
  arrange(result)
# check there is no 'na' abundance
merman_data_hab_cs %>%
  as.data.frame() %>%
  filter(!is.na(result))

abundance contains records of ‘presence’, result includes numerical counts

  • check the unusual results records
merman_data_hab_cs %>%
  as.data.frame() %>%
  filter(result == "#MULTIVALUE"| result == "0") %>%
  arrange(result)

#MULTIVALUE is only recorded for COPEPODA, we’ll filter these records out as records of COPEPODA will not be retained in the final analysis

merman_data_hab_cs <- merman_data_hab_cs %>%
  filter(result != "#MULTIVALUE")

0 result counts are recorded with Presence

  • Check if other count values are recorded with Presence
merman_data_hab_cs %>%
  as.data.frame() %>%
  filter(abundance == "Presence") %>%
  arrange(desc(result))

No other counts with Presence records

  • update 0s to NA
merman_data_hab_cs <- merman_data_hab_cs %>%
  mutate(result = na_if(result, "0"))
  • Check for multiple records of the same taxa with different abundance records (in the same sample)
# find all rows with duplicates in every field except results or abundance
dups <- merman_data_hab_cs %>%
  as.data.frame() %>%
  select(-c(result, abundance, geometry)) %>%
  duplicated()
# find the fields to filter the data
dup_list <- merman_data_hab_cs %>%
  as.data.frame() %>%
  filter(dups) %>%
  select(sampleref, samplestartdate, specieslatinname)

# Show duplicates
merman_data_hab_cs %>%
  as.data.frame() %>%
  right_join(dup_list)

Whilst there is no duplicates with count records and Presence records, there are duplicate counts for Clymenura in these five samples

  • Check result doesn’t contain decimals
  • Convert result to numerical
  • Add count value of 1 for Presence records
  • Remove abundancefield
  • Sum counts of duplicate records
  • update count values and remove duplicates
# check for decimals
merman_data_hab_cs %>%
  as.data.frame() %>%
  mutate(result = as.double(result)) %>%
  filter(!result%%1==0)
## Warning in `[<-.data.frame`(`*tmp*`, is_list, value = list(`21` =
## "<s_GEOMET>")): replacement element 1 has 1 row to replace 0 rows
# change to numerical type
merman_data_hab_cs <- merman_data_hab_cs %>%
  mutate(result = as.double(result))
# update Presence to 1
merman_data_hab_cs <- merman_data_hab_cs %>%
  mutate(result = if_else(abundance == "Presence", 1, result))
# remove "Presence" field
merman_data_hab_cs <- merman_data_hab_cs %>%
  select(-c(abundance))

# sum duplicate records
qc_temp <- merman_data_hab_cs %>%
  as.data.frame() %>%
  right_join(dup_list) %>%
  mutate(result = as.double(result)) %>%
  group_by(sampleref, samplestartdate, specieslatinname) %>%
  summarise(count_sum = sum(result))

# add sum values and remove duplicates
merman_data_hab_cs <- merman_data_hab_cs %>%
  left_join(qc_temp) %>%
  filter(!dups) %>%
  mutate(result = if_else(is.na(count_sum), result, count_sum)) %>%
  select(-c(count_sum))

# tidy up
rm(dups, dup_list, qc_temp)

Numeric Count Abundance Records

check the distribution of count records in the `result`` field for any unusual frequencies/values

merman_data_hab_cs  %>%
  as.data.frame() %>%
    ggplot() +
        geom_histogram(
            mapping = aes(result),
            colour = "black",
            breaks = seq(from = 0, to = 30, by = 1)) +
    scale_y_continuous(name = "frequency") +
    scale_x_continuous(breaks = c(1:30))

merman_data_hab_cs  %>%
  as.data.frame() %>%
    ggplot() +
        geom_histogram(
            mapping = aes(result),
            colour = "black",
            breaks = seq(from = 30, to =150, by = 1)) +
    scale_y_continuous(name = "frequency", breaks = seq(0, 60, 5)) +
    scale_x_continuous(breaks = seq(30, 150, 10))

merman_data_hab_cs  %>%
  as.data.frame() %>%
    ggplot() +
        geom_histogram(
            mapping = aes(result),
            colour = "black",
            breaks = seq(from = 150, to = 2600, by = 1)) +
    scale_y_continuous(name = "frequency", breaks = seq(0, 5, 1)) +
    scale_x_continuous(breaks = seq(100, 2600, 200))

As expected, frequency of abundance values rapidly declines with higher values

  • check the higher abundance values for any unusual records
merman_data_hab_cs  %>%
  as.data.frame() %>%
  select(sampleref, monitoringyear, specieslatinname, stageofdevelopment, result, msfd) %>%
  filter(result > 250) %>%
  arrange(specieslatinname)

Although some counts are high, none appear to be erroneous. The record of 2500 Gobiidae eggs will be removed in the next stages.


Duplicate Taxa with Different Qualifiers

  • First check for records that can be removed based on quailier (e.g. records of ‘eggs’ or ‘dead’ specimens)
# 1st qualifier
merman_data_hab_cs %>%
  as.data.frame() %>%
  group_by(stageofdevelopment) %>%
  count(stageofdevelopment)
# 2nd qualifier
merman_data_hab_cs %>%
  as.data.frame() %>%
  group_by(sexofspecies) %>%
  count(sexofspecies)
  • Remove records with Egg qualifier
# remove eggs
merman_data_hab_cs <- merman_data_hab_cs %>%
  filter(grepl("Egg", stageofdevelopment) == FALSE)
  • Check for duplicate taxa recorded with different qualifiers
dups <- merman_data_hab_cs %>%
  as.data.frame() %>%
  select(-c(stageofdevelopment, sexofspecies)) %>%
  duplicated()

# find the fields to filter the data
dup_list <- merman_data_hab_cs %>%
  as.data.frame() %>%
  filter(dups) %>%
  select(sampleref, samplestartdate, specieslatinname)

# Show duplicates
merman_data_hab_cs %>%
  as.data.frame() %>%
  right_join(dup_list)
## Joining, by = c("sampleref", "samplestartdate", "specieslatinname")
  • Aggregate the abundance values for these records
  • Remove the qualifier fields (stageofdevelopment & sexofspecies)
# sum duplicate records
qc_temp <- merman_data_hab_cs %>%
  as.data.frame() %>%
  right_join(dup_list) %>%
  mutate(result = as.double(result)) %>%
  group_by(sampleref, samplestartdate, specieslatinname) %>%
  summarise(count_sum = sum(result))

# add sum values and remove duplicates
merman_data_hab_cs <- merman_data_hab_cs %>%
  left_join(qc_temp) %>%
  filter(!dups) %>%
  mutate(result = if_else(is.na(count_sum), result, count_sum)) %>%
  select(-c(count_sum, stageofdevelopment, sexofspecies))

# tidy up
rm(dups, dup_list, qc_temp)

  • sampleref is not a unique sample identifier (combination of station name and replicate name)
  • Add a unique samplecode field (combination of sampleref and samplestartdate)
merman_samples_count <- merman_data_hab_cs %>%
  as.data.frame() %>%
  mutate(samplecode = paste(sampleref, samplestartdate, sep = "_"))

Save Processed Data

Save the MERMAN data for processing in the next script

# Processed MERMAN data
saveRDS(merman_samples_count, paste0("Data/Processed/R_data/", creation_date(), "_merman_samples_count"))

Summary of Gear Types, Habitats and Years sampled

Visualise the gear types and habitats across years in the different datasets following initial filtering and cleaning.

  • First pull the key fields from the datasets into one table
# take a summary of MR samples
MR_temp <- MR_samples_count %>%
  as.data.frame() %>%
  group_by(samplekey, sampledate, sampleyear, geartype, msfd) %>%
  dplyr::summarise(n = n(), .groups = "drop") %>%
  mutate(dataset = "MR")
# take a summary of OB samples
OB_temp <- OB_samples_count %>%
  as.data.frame() %>%
  group_by(samplecode, sampledate, year, gearname, msfd) %>%
  dplyr::summarise(n = n(), .groups = "drop") %>%
  dplyr::rename(samplekey = samplecode, sampleyear = year, geartype = gearname) %>%
  mutate(dataset = "OB")
# take a summary of WFD samples
WFD_temp <- WFD_samples_count %>%
  as.data.frame() %>%
  group_by(samplecode, sampledate, sampleyear, samplemethod, msfd) %>%
  dplyr::summarise(n = n(), .groups = "drop") %>%
  dplyr::rename(samplekey = samplecode, geartype = samplemethod) %>%
  mutate(dataset = "WFD")
# take a summary of MERMAN samples
merman_temp <- merman_samples_count %>%
  as.data.frame() %>%
  group_by(samplecode, samplestartdate, monitoringyear, samplertype, msfd) %>%
  dplyr::summarise(n = n(), .groups = "drop") %>%
  dplyr::rename(samplekey = samplecode, sampleyear = monitoringyear, sampledate = samplestartdate, geartype = samplertype) %>%
  mutate(dataset = "MERMAN",
         sampledate = lubridate::dmy(sampledate))
# take a summary of NRW samples (samples were mostly Day grabs)
NRW_temp <- samples_NRW_bht %>%
  select(sampleref, sampledate, sampleyear, msfd) %>%
  mutate(geartype = "Day") %>%
  dplyr::rename(samplekey = sampleref) %>%
  mutate(dataset = "NRW")
# combine sample summaries
combined_temp <- bind_rows(MR_temp, OB_temp, WFD_temp, merman_temp, NRW_temp)
# tidy up
rm(MR_temp, OB_temp, WFD_temp, merman_temp, NRW_temp)
  • Create a summary of gear type used across years and datasets
# standardise gear type names
combined_temp <- combined_temp %>%
  as.data.frame() %>%
  mutate(geartype = case_when(grepl("day", geartype, ignore.case = T) ~ "Day Grab",
                              grepl("hamon", geartype, ignore.case = T) ~ "Hamon Grab",
                              grepl("Veen", geartype, ignore.case = T) ~ "Van Veen Grab",
                              TRUE ~ geartype))
# pull a list of gear names
gearlist <- combined_temp %>%
    chuck("geartype") %>%
    unique %>%
    sort
# summary of sample methods and years
combined_temp %>%
    ggplot() +
        geom_jitter(
            mapping = aes(x = sampledate, y = match(geartype, gearlist), colour = dataset),
            alpha = 0.5, width = 0, height = 0.2) +
        scale_x_date(name = "", date_breaks = "1 years", date_labels = "%Y") +
        scale_y_continuous(name = "", breaks = seq_len(length(gearlist)), labels = gearlist) +
        scale_color_manual(values = cb_palette) +
        labs(title = "Gear Types") +
        theme(legend.title=element_blank()) +
        theme_bw() +
        guides(color=guide_legend(title="Dataset", override.aes = list(size=3)))

# clean up
rm(gearlist)
  • Create a summary of broad-scale habitats across years and datasets
hablist <- combined_temp %>%
    chuck("msfd") %>%
    unique %>%
    sort(decreasing = TRUE) %>%
    str_replace("and/or", "and/or\n")

# summary of samples and years
combined_temp %>%
  mutate(msfd = str_replace(msfd, "and/or", "and/or\n")) %>%
    ggplot() +
        geom_jitter(
            mapping = aes(x = sampledate, y = match(msfd, hablist), colour = dataset),
            alpha = 0.5, width = 0, height = 0.2) +
        scale_color_manual(values = cb_palette) +
        scale_x_date(name = "", date_breaks = "1 years", date_labels = "%Y") +
        scale_y_continuous(name = "", breaks = seq_len(length(hablist)), labels = hablist) +
        labs(title = "Broad-scale Habitats Sampled") +
        theme(legend.title=element_blank()) +
        theme_bw() +
        guides(color=guide_legend(title="Dataset", override.aes = list(size=3)))

# tidy up 
rm(hablist)

Data preparation and analysis is continued in the next script: Data Truncation

Session Info

details::details(sessionInfo(), summary = 'sessionInfo')
sessionInfo

R version 3.6.1 (2019-07-05)
Platform: x86_64-w64-mingw32/x64 (64-bit)
Running under: Windows Server x64 (build 14393)

Matrix products: default

locale:
[1] LC_COLLATE=English_United Kingdom.1252 
[2] LC_CTYPE=English_United Kingdom.1252   
[3] LC_MONETARY=English_United Kingdom.1252
[4] LC_NUMERIC=C                           
[5] LC_TIME=English_United Kingdom.1252    

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base     

other attached packages:
 [1] RColorBrewer_1.1-2 details_0.3.0      DT_0.26            sf_0.9-8          
 [5] forcats_0.5.1      stringr_1.4.0      dplyr_1.0.7        purrr_0.3.4       
 [9] readr_2.1.0        tidyr_1.1.4        tibble_3.1.6       ggplot2_3.3.5     
[13] tidyverse_1.3.1    rprojroot_2.0.3   

loaded via a namespace (and not attached):
 [1] httr_1.4.2         sass_0.4.4         jsonlite_1.7.2     modelr_0.1.8      
 [5] bslib_0.4.1        assertthat_0.2.1   highr_0.9          cellranger_1.1.0  
 [9] yaml_2.2.1         pillar_1.6.4       backports_1.3.0    glue_1.5.0        
[13] digest_0.6.28      rvest_1.0.2        colorspace_2.0-2   htmltools_0.5.2   
[17] clipr_0.7.1        pkgconfig_2.0.3    broom_0.7.10       haven_2.4.3       
[21] s2_1.0.4           scales_1.1.1       tzdb_0.2.0         proxy_0.4-27      
[25] generics_0.1.1     farver_2.1.0       ellipsis_0.3.2     cachem_1.0.6      
[29] withr_2.5.0        cli_3.1.0          magrittr_2.0.1     crayon_1.4.2      
[33] readxl_1.3.1       evaluate_0.18      fs_1.5.0           fansi_0.5.0       
[37] xml2_1.3.2         class_7.3-15       tools_3.6.1        hms_1.1.1         
[41] lifecycle_1.0.1    munsell_0.5.0      reprex_2.0.1       compiler_3.6.1    
[45] jquerylib_0.1.4    e1071_1.7-11       rlang_0.4.12       classInt_0.4-3    
[49] units_0.8-0        grid_3.6.1         rstudioapi_0.13    htmlwidgets_1.5.4 
[53] crosstalk_1.2.0    labeling_0.4.2     rmarkdown_2.18     wk_0.6.0          
[57] gtable_0.3.0       DBI_1.1.3          R6_2.5.1           lubridate_1.8.0   
[61] knitr_1.41         fastmap_1.1.0      utf8_1.2.2         KernSmooth_2.23-15
[65] desc_1.4.2         stringi_1.7.5      Rcpp_1.0.10        vctrs_0.3.8       
[69] png_0.1-7          dbplyr_2.1.1       tidyselect_1.1.1   xfun_0.35