Updated / printed: 2023-09-07
This script continues the data preparation methods needed for the Condition of Benthic Communities indicator (BH2b) in the UK EEZ Celtics Seas areas. Please see Data Preparation for previous data preparation steps.
This script focuses on;
## Load in the OneBenthic, Marine Recorder and WFD data from the previous script
# OneBenthic
OB_data_hab_cs <- readRDS("Data/Processed/R_data/C20230816_OB_data_hab_cs_bht")
# Marine Recorder
MR_data_hab_cs <- readRDS("Data/Processed/R_data/C20230823_MR_data_hab_cs_bht")
# WFD
WFD_data_hab_cs <- readRDS("Data/Processed/R_data/C20230526_WFD_data_hab_cs_bht")
# MEREMAN
merman_data_hab_cs <- readRDS("Data/Processed/R_data/C20230816_merman_data_hab_cs_bht")
# NRW
samples_NRW_bht <- readRDS("Data/Processed/R_data/C20230831_samples_NRW_bht")
Sample diversity will vary with the sieve mesh size used for the analysis, for comparable results (reference values) all samples within an assessment group (i.e., grab-core) should have the same mesh size.
In the CEMP guidelines (‘3.2 Quantitative Objectives’) sampling categories are suggested, for Grab-Core these are given as:
In previous drafts of data preparation both 0.5 mm and 1 mm mesh sizes samples were retained in the dataset. This was with the intention that 0.5 mm mesh size samples could potentially be assessed in addition to 1mm samples. There currently isn’t scope to include 0.5 mm mesh size samples at this time so they will be removed from the dataset
# Check the sample mesh size
OB_data_hab_cs %>%
as.data.frame() %>%
select(samplecode, gearname, macrosievesizemm) %>%
group_by(samplecode, gearname, macrosievesizemm) %>%
dplyr::summarise(count = n_distinct(samplecode)) %>%
select(gearname, macrosievesizemm, count) %>%
group_by(gearname, macrosievesizemm) %>%
dplyr::summarise(total = sum(count), .groups = "drop") %>%
datatable(
extensions = "Buttons",
escape = FALSE,
options = list(dom = "Blfrtip", buttons = c("csv")))
There are some unusual mesh sizes (>1)
# Mesh >1
OB_data_hab_cs %>%
as.data.frame() %>%
select(surveyname, gearname, macrosievesizemm, validnameworms, abund) %>%
filter(macrosievesizemm >1)
Records with unusual mesh sizes do not have available data
(data witheld)
OB_data_hab_cs_m <- OB_data_hab_cs %>%
filter(macrosievesizemm == 1)
After filtering mesh size;
# Check the sample mesh size
MR_data_hab_cs %>%
as.data.frame() %>%
select(samplekey, geartype, meshsize) %>%
group_by(samplekey, geartype, meshsize) %>%
dplyr::summarise(count = n_distinct(samplekey)) %>%
select(geartype, meshsize, count) %>%
group_by(geartype, meshsize) %>%
dplyr::summarise(total = sum(count), .groups = "drop") %>%
datatable(
extensions = "Buttons",
escape = FALSE,
options = list(dom = "Blfrtip", buttons = c("csv")))
Frustratingly there are many samples without mesh size recorded, Day grabs in particular have 1587 samples without mesh size (1153 0.5mm and 1344 1mm).
# Load UKMS assessment units
ukms <- st_read("Data/Raw/UKMS/C20230620_UKMS_final.shp")
# fix geometry
ukms <- st_make_valid(ukms)
# filter to Celtic Seas
ukms_cs <- filter(ukms, UKMS_SubRe == "Celtic Seas")
# Read in world landmass shapefile for creating maps
land <- st_read(paste0(NEA_PANACEA_folder, "Benthic Habitat AUs and Land Shapefiles/Benthic Habitat AUs and Land Shapefiles/c20210824_World_Outline_WGS84_LAEA.shp"))
# transform to WGS84
land <- st_transform(land, 4326)
# fix geometry
land <- st_make_valid(land)
# clip the land to Celtic Seas area
sf::sf_use_s2(FALSE)
land_cs <- st_crop(land, ukms_cs)
sf::sf_use_s2(TRUE)
# Filter to samples without mesh size info, group to sample and drop unnecessary fields
MR_data_na_mesh <- MR_data_hab_cs %>%
filter(is.na(meshsize)) %>%
group_by(surveykey, samplekey, sampledate, sampleyear, geartype, msfd, eunis_l3, hab_type) %>%
dplyr::summarise(N = sum(spcount, na.rm = TRUE), S = n_distinct(preferredspeciesname), .groups = "drop")
# Plot samples
MR_data_na_mesh %>%
ggplot() +
geom_sf(data = land_cs, fill = "grey", colour = NA) +
geom_sf(data = ukms_cs, mapping = aes(geometry = geometry), lwd = 0.5, colour = "grey40", fill = NA) +
geom_sf(mapping = aes(geometry = geometry), colour = "blue", size = 1.2) +
ylim(49.9, 60) +
xlim(-9, -1) +
theme_bw() +
theme(legend.position = "none") +
ggtitle("Marine Recorder Samples Without Mesh Size")
# load in MR survey names
surveyname <- read.csv("Data/Processed/Marine_Recorder/SurveyName.csv")
# fix field name
surveyname <- surveyname %>%
rename("SurveyKey" = "ï..SurveyKey") %>%
mutate(SurveyKey = as.character(SurveyKey),
SurveyName = as.character(SurveyName))
# add survey names to MR samples
MR_data_na_mesh_surveys <- left_join(MR_data_na_mesh, surveyname, by = c("surveykey" = "SurveyKey"))
# summarise surveys with NA mesh
MR_data_na_mesh_surveys %>%
as.data.frame() %>%
group_by(SurveyName, surveykey, geartype) %>%
dplyr::summarise(sample_n = n()) %>%
datatable(
caption = "MR survey data with NA mesh size",
colnames = c("samples" = "sample_n"),
extensions = "Buttons",
escape = FALSE,
rownames = FALSE,
options = list(dom = "Blfrtip", buttons = c("csv")))
The summary of survey without analysis mesh size metadata was shared with the Benthic Sub-Group (BSG) to fill in data gaps. Additionally, survey details available on the MEDIN portal were interrogated for evidence of the analysis methodology.
The following surveys were identified as having samples processed with a 1 mm sieve mesh:
# list of survey to be considered for analysis (1 mm)
surveys_1mm <- c("2010 SNH survey to establish the distribution of Priority Marine Features in the Clyde Sea area",
"2011 INIS Hydro Grab Survey",
"2011 Marine Scotland North Minch benthic infaunal survey",
"2011 Marine Scotland Small Isles benthic infaunal survey",
"2011 Marine Scotland survey of Lochs Linnhe, Etive, Leven and Eil: a broadscale validation - part A",
"2012 Belfast Ecosystem Assessment",
"2012 CEFAS Mounts Bay Video and Grab Survey",
"2012 Defra The Manacles rMCZ Site Verification Survey",
"2012-2013 AFBI Essential Fish Habitat Surveys",
"2013 Natural England Morecambe and Lune (Walney Channel) Benthic Grab Survey",
"2014 AFBI Fair Head Tidal Energy Development Site Survey",
"2014 AFBI Rathlin Deeps",
"2014-2015 AFBI Irish Sea Mud Grab Survey",
"2014-2016 AFBI Outer Ards Modiolus Surveys",
"2017 AFBI North Channel Habitat Mapping Project Benthic Infauna")
# Summary of surveys ad samples (with count data)
MR_data_na_mesh_surveys %>%
as.data.frame() %>%
filter(SurveyName %in% surveys_1mm, N > 0) %>%
group_by(SurveyName) %>%
tally()
N > 0)meshsize# take a list of sample keys for update
samplekey_1mm <- MR_data_na_mesh_surveys %>%
as.data.frame() %>%
filter(SurveyName %in% surveys_1mm, N > 0) %>%
pull(samplekey)
# update records to 1 mm
MR_data_hab_cs <- MR_data_hab_cs %>%
mutate(meshsize = if_else(samplekey %in% samplekey_1mm, 1.0, meshsize))
# tidy up
rm(samplekey_1mm, MR_data_na_mesh_surveys, surveys_1mm)
MR_data_hab_cs_m <- MR_data_hab_cs %>%
filter(meshsize == 1)
After filtering mesh size;
# Check the sample mesh size
WFD_data_hab_cs %>%
as.data.frame() %>%
select(samplecode, samplemethod, sievemeshum) %>%
group_by(samplecode, samplemethod, sievemeshum) %>%
dplyr::summarise(count = n_distinct(samplecode)) %>%
select(samplemethod, sievemeshum, count) %>%
group_by(samplemethod, sievemeshum) %>%
dplyr::summarise(total = sum(count), .groups = "drop") %>%
datatable(
extensions = "Buttons",
escape = FALSE,
options = list(dom = "Blfrtip", buttons = c("csv")))
WFD_data_hab_cs_m <- WFD_data_hab_cs %>%
filter(sievemeshum == 1000)
After filtering mesh size;
The datasets still include samples with either no habitat information or not enough information to translate to broad-scale habitat definitions.
No habitat dataNo EUNIS to MSFD BBHT translation# filter MR
MR_data_hab_cs_m <- MR_data_hab_cs_m %>%
filter(msfd != "No habitat data", msfd != "No EUNIS to MSFD BBHT translation")
# summary of BHTs
MR_data_hab_cs_m %>%
as.data.frame() %>%
group_by(samplekey, msfd) %>%
summarise() %>%
group_by(msfd) %>%
count()
After filtering;
# filter OB
OB_data_hab_cs_m <- OB_data_hab_cs_m %>%
filter(msfd != "No habitat data", msfd != "No EUNIS to MSFD BBHT translation")
# summary of BHTs
OB_data_hab_cs_m %>%
as.data.frame() %>%
group_by(samplecode, msfd) %>%
summarise() %>%
group_by(msfd) %>%
count()
After filtering:
# filter
WFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
filter(msfd != "No habitat data", msfd != "No EUNIS to MSFD BBHT translation")
# summary of BHTs
WFD_data_hab_cs_m %>%
as.data.frame() %>%
group_by(samplecode, msfd) %>%
summarise() %>%
group_by(msfd) %>%
count()
After filtering:
# filter
samples_NRW_bht <- samples_NRW_bht %>%
filter(msfd != "No habitat data", msfd != "No EUNIS to MSFD BBHT translation")
# summary of BHTs
samples_NRW_bht %>%
as.data.frame() %>%
group_by(msfd) %>%
count()
After filtering:
# filter
merman_data_hab_cs <- merman_data_hab_cs %>%
filter(msfd != "No habitat data", msfd != "No EUNIS to MSFD BBHT translation")
# summary of BHTs
merman_data_hab_cs %>%
as.data.frame() %>%
group_by(sampleref, msfd) %>%
summarise() %>%
group_by(msfd) %>%
count()
After filtering:
A number of samples from each dataset are assigned rock and biogenic reef broad-scale habitats, either from sample information (Marine Recorder samples) or from spatially joining to the habitat map. As sampling has been preformed by means of grabs or cores the assignment of hard substrata is inappropriate, following the same approach to the QSR 2023 assessment these sample will be assigned to mixed sediment.
We know that rock and biogenic reef can’t be present in the data, as sampling has been performed by means of grabs, cores, dredges, and trawls. We will therefore rename “rock and biogenic reef” to “mixed sediment” (e-mail Sander 2022-07-18T12:13).
step-2-eda.html (BH2b in the Greater North Sea script)
Note, with this approach there are some samples with biogenic habitat metadata (e.g. A5.6 Sublittoral biogenic reefs) assigned to the mixed sediment broad-scale habitat groups.
MR_data_hab_cs_m %>%
as.data.frame() %>%
filter(grepl("A5.6", hab_type)) %>%
group_by(samplekey, hab_type, msfd) %>%
count() %>%
group_by(hab_type, msfd) %>%
count()
MR_data_hab_cs_m <- MR_data_hab_cs_m %>%
mutate(msfd = str_replace(msfd, "rock and biogenic reef", "mixed sediment"))
OB_data_hab_cs_m %>%
as.data.frame() %>%
filter(grepl("A5.6", hab_type)) %>%
group_by(samplecode, hab_type, msfd) %>%
count() %>%
group_by(hab_type, msfd) %>%
count()
OB_data_hab_cs_m <- OB_data_hab_cs_m %>%
mutate(msfd = str_replace(msfd, "rock and biogenic reef", "mixed sediment"))
WFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
mutate(msfd = str_replace(msfd, "rock and biogenic reef", "mixed sediment"))
merman_data_hab_cs %>%
as.data.frame() %>%
filter(grepl("A5.6", hab_type)) %>%
group_by(sampleref, hab_type, msfd) %>%
count() %>%
group_by(hab_type, msfd) %>%
count()
merman_data_hab_cs <- merman_data_hab_cs %>%
mutate(msfd = str_replace(msfd, "rock and biogenic reef", "mixed sediment"))
samples_NRW_bht %>%
as.data.frame() %>%
filter(grepl("A5.6", hab_type)) %>%
group_by(sampleref, hab_type, msfd) %>%
count() %>%
group_by(hab_type, msfd) %>%
count()
samples_NRW_bht <- samples_NRW_bht %>%
mutate(msfd = str_replace(msfd, "rock and biogenic reef", "mixed sediment"))
# Processed NRW data
saveRDS(samples_NRW_bht, paste0("Data/Processed/R_data/", creation_date(), "_samples_NRW_bht"))
The datasets need field names and data types standardising (count abundance) and any erroneous or duplicated data removing. Taxa and qualifier records also needed amalgamating to taxa level (removing qualifiers).
Common fields and equivalents between datasets:
| Marine Recorder | OneBenthic | EA WFD | MERMAN | Description |
|---|---|---|---|---|
samplekey |
samplecode |
samplecode |
sampleref |
sample ID |
meshsize |
macrosievesizemm |
sievemeshum |
meshsize |
mesh size mm/um |
sampledate |
sampledate |
sampledate |
samplestartdate |
raw sample date |
sampleyear |
year |
sampleyear |
monitoringyear |
year sampled |
speciesname |
validnameworms |
taxonname |
specieslatinname |
taxa name |
preferredspeciesname |
validnameworms |
preferredtaxonname |
specieslatinname |
WoRMS taxa name / NHM name |
preferredaphiaid |
aphiaid |
taxonlistitemkey |
NA | WoRMS APHIA ID / NHM ID |
spqual |
taxonqualifier |
qualifier |
stageofdevelopment/sexofspecies |
taxa qualifier |
sacforn |
NA | NA | NA | SACFOR abundance |
sppercent |
NA | NA | NA | percentage abundance |
pa |
abund |
abundance |
abundance |
presents / absence |
spcount |
abund |
abundance |
results |
abundance count |
eunis |
NA | NA | NA | sample EUNIS code (Marine Recorder metadata) |
hab_type |
hab_type |
hab_type |
hab_type |
EUNIS from combined map |
msfd |
msfd |
msfd |
msfd |
broad-scale habitat definitions |
# find all rows with duplicates in every field
dups <- MR_data_hab_cs_m %>%
as.data.frame() %>%
duplicated()
# list of duplicates
MR_data_hab_cs_m %>%
as.data.frame() %>%
filter(dups) %>%
select(samplekey, geartype, speciesname, spcount) %>%
arrange(samplekey, speciesname)
# filter duplicates
MR_data_hab_cs_m <- MR_data_hab_cs_m %>%
filter(dups == FALSE)
# tidy up
rm(dups)
The Marine Recorder data includes different measures of occurrence
MR_data_hab_cs_m %>%
as.data.frame() %>%
select(sacforn) %>%
group_by(sacforn) %>%
dplyr::summarise(n = n(), .groups = "drop")
MR_data_hab_cs_m %>%
as.data.frame() %>%
select(pa) %>%
group_by(pa) %>%
dplyr::summarise(n = n(), .groups = "drop")
MR_data_hab_cs_m %>%
as.data.frame() %>%
select(sppercent) %>%
group_by(sppercent) %>%
dplyr::summarise(n = n(), .groups = "drop")
MR_data_hab_cs_m %>%
as.data.frame() %>%
select(numpersqm) %>%
group_by(numpersqm) %>%
dplyr::summarise(n = n(), .groups = "drop")
number per sqm and spcount occur
on independentlyMR_data_hab_cs_m %>%
as.data.frame() %>%
select(spcount, numpersqm) %>%
filter(is.na(spcount) & !is.na(numpersqm))
sppercent field# remove species percentage field
MR_data_hab_cs_m <- MR_data_hab_cs_m %>%
select(-c(sppercent))
pa
field
numpersqm is always present with spcount,
preference to use spcount
numpersqm# trim and remove per sq m
MR_data_hab_cs_m <- MR_data_hab_cs_m %>%
mutate(pa = str_trim(pa)) %>%
select(-c(numpersqm))
# group by record type
MR_data_hab_cs_m %>%
as.data.frame() %>%
select(sacforn, pa, spcount) %>%
mutate(count_included = if_else(is.na(spcount),"no","yes")) %>%
group_by(sacforn, pa, count_included) %>%
dplyr::summarise(n = n(), .groups = "drop")
1 later, as per CEMP guidelines)
# convert p to P
MR_data_hab_cs_m <- MR_data_hab_cs_m %>%
mutate(pa = if_else(pa == "p", "P", pa))
Taxa/qualifiers without abundance:
MR_data_hab_cs_m %>%
as.data.frame() %>%
select(speciesname, spqual, spcount, sacforn, pa) %>%
filter(is.na(spcount), sacforn == "" | is.na(sacforn), pa == "" | is.na(pa)) %>%
group_by(speciesname, spqual) %>%
dplyr::summarise(n = n(), .groups = "drop") %>%
arrange(speciesname) %>%
datatable(
caption = "Taxa without abundance records",
extensions = "Buttons",
rownames = FALSE,
escape = FALSE,
options = list(dom = "Blfrtip", buttons = c("csv")))
QC a random selection of records without abundance:
MR_data_hab_cs_m %>%
as.data.frame() %>%
select(samplekey, speciesname, spqual, sacforn, pa, spcount, msfd) %>%
filter(is.na(spcount), sacforn == "" | is.na(sacforn), pa == "" | is.na(pa)) %>%
slice_sample(n = 20)
MR_data_hab_cs_m %>%
as.data.frame() %>%
select(samplekey, geartype, sampledate, speciesname, spqual, sacforn, pa, spcount) %>%
filter(samplekey == "MRNE015400002F35") %>%
arrange(speciesname) %>%
datatable(
extensions = "Buttons",
rownames = FALSE,
escape = FALSE,
options = list(dom = "Blfrtip", buttons = c("csv")))
In this example sample (MRNE015400002F35) there are many rows without abundance, however there are records with counts and “Present” records. This suggests not all missing values are “Present”. There are also instances of taxa at multiple levels of classification (e.g. Amathia lendigera & Amathia) without abundance, it’s unlikely these taxa would be split like this in the same sample.
Check if the missing values are from a specific survey
MR_data_hab_cs_m %>%
as.data.frame() %>%
filter(is.na(spcount), sacforn == "" | is.na(sacforn), pa == "" | is.na(pa)) %>%
group_by(surveyeventkey) %>%
dplyr::summarise(samples = n_distinct(samplekey), .groups = "drop")
# total of all samples
all <- MR_data_hab_cs_m %>%
as.data.frame() %>%
mutate(survey_part = str_sub(surveyeventkey, 0, 8)) %>%
group_by(survey_part, sampleyear) %>%
dplyr::summarise(samples_total = n_distinct(samplekey), .groups = "drop")
# samples missing abundance values
filtered <- MR_data_hab_cs_m %>%
as.data.frame() %>%
filter(is.na(spcount), sacforn == "" | is.na(sacforn), pa == "" | is.na(pa)) %>%
mutate(survey_part = str_sub(surveyeventkey, 0, 8)) %>%
group_by(survey_part, sampleyear) %>%
dplyr::summarise(samples = n_distinct(samplekey), .groups = "drop")
# summary table
missing_values <- left_join(filtered, all, by = c("survey_part", "sampleyear"))
missing_values %>%
arrange(survey_part, sampleyear)
# tidy up
rm(all, filtered, missing_values)
Filter out the taxa records missing abundance and keep the remaining as legitimate records.
# for reference, take a list of samples with missing abundance records
MR_samples_abun_na <- MR_data_hab_cs_m %>%
as.data.frame() %>%
filter(is.na(spcount), sacforn == "" | is.na(sacforn), pa == "" | is.na(pa)) %>%
select(samplekey) %>%
unique() %>%
pull()
# take the list as a temp df for joining
qc <- MR_data_hab_cs_m %>%
as.data.frame() %>%
filter(is.na(spcount), sacforn == "", pa == "") %>%
select(samplekey) %>%
unique()
# join on samples missing abundance
qc_temp <- left_join(qc, MR_data_hab_cs_m, by = c("samplekey" = "samplekey"), suffix=c("",".y"))
# summarise missing abundance
qc_na <- qc_temp %>%
as.data.frame() %>%
group_by(samplekey) %>%
dplyr::summarise(count_null = n(), .groups = "drop")
# summarise those with abundance
qc_abun <- qc_temp %>%
as.data.frame() %>%
filter(!is.na(spcount)| sacforn != ""| pa != "") %>%
group_by(samplekey) %>%
dplyr::summarise(count_abun = n(), .groups = "drop")
# combine missing and those with abundance
left_join(qc_na, qc_abun, by = c("samplekey" = "samplekey"), suffix=c("",".y")) %>%
datatable(
extensions = "Buttons",
rownames = FALSE,
escape = FALSE,
options = list(dom = "Blfrtip", buttons = c("csv")))
# tidy up temp data
rm(MR_samples_abun_na, qc, qc_temp, qc_na, qc_abun)
# filter rows with NA in all abundance fields
MR_samples_abun_m <- MR_data_hab_cs_m %>%
filter(!is.na(spcount) | !is.na(sacforn) | !is.na(pa))
# filter empty SACFOR or P/A rows (without null count)
MR_samples_abun_m <- MR_samples_abun_m %>%
filter(!is.na(spcount) | sacforn != "" | pa != "")
pa) had both
P and N valuesMR_samples_abun_m %>%
as.data.frame() %>%
select(samplekey, sampleyear, meshsize, preferredspeciesname, spqual, pa) %>%
filter(pa == "N") %>%
arrange(preferredspeciesname)
samplekey starting MRSNH029000003)first 100 rows of MRSNH029000003 data:
MR_samples_abun_m %>%
as.data.frame() %>%
select(samplekey, sampleyear, preferredspeciesname, spqual, pa, spcount, sacforn) %>%
filter(grepl("MRSNH029000003", samplekey)) %>%
arrange(preferredspeciesname) %>%
arrange(samplekey) %>%
slice_head(n=100)
MR_samples_abun_m %>%
as.data.frame() %>%
select(samplekey, pa) %>%
filter(pa == "N") %>%
distinct(samplekey)
MR_samples_abun_m <- MR_samples_abun_m %>%
filter(pa != "N")
MR_samples_abun_m %>%
as.data.frame() %>%
select(samplekey, geartype, sampleyear, meshsize, preferredspeciesname, spqual, sacforn, pa, spcount) %>%
filter(!is.na(sacforn) & sacforn != "") %>%
arrange(preferredspeciesname)
MR_samples_abun_m %>%
as.data.frame() %>%
select(samplekey, geartype, sampleyear, meshsize, preferredspeciesname, spqual, sacforn, pa, spcount) %>%
filter(samplekey == "MRNE01080000761D") %>%
arrange(preferredspeciesname)
sacforn fieldMR_samples_abun_m <- MR_samples_abun_m %>%
filter(samplekey != "MRNE01080000761D") %>%
select(-c(sacforn))
spcount field includes fractions/decimal
places (e.g. if samples have been sub-sampled and not rounded to
integers)MR_samples_abun_m %>%
as.data.frame() %>%
filter(!spcount%%1==0)
spcount
field for any unusual frequenciesMR_samples_abun_m %>%
as.data.frame() %>%
filter(spcount <11) %>%
ggplot() +
geom_histogram(
mapping = aes(spcount),
colour = "black",
breaks = seq(from = 0, to = 10, by = 1)) +
scale_y_continuous(name = "frequency") +
scale_x_continuous(breaks = c(1:10))
MR_samples_abun_m %>%
as.data.frame() %>%
filter(spcount >10 & spcount <101) %>%
ggplot() +
geom_histogram(
mapping = aes(spcount),
colour = "black",
breaks = seq(from = 10, to = 100, by = 1)) +
scale_y_continuous(name = "frequency") +
scale_x_continuous(breaks = seq(10, 100, 10))
MR_samples_abun_m %>%
as.data.frame() %>%
filter(spcount >100 & spcount <1001) %>%
ggplot() +
geom_histogram(
mapping = aes(spcount),
colour = "black",
breaks = seq(from = 100, to = 1000, by = 1)) +
scale_y_continuous(name = "frequency") +
scale_x_continuous(breaks = seq(100, 1000, 100))
MR_samples_abun_m %>%
as.data.frame() %>%
filter(spcount >1000) %>%
ggplot() +
geom_histogram(
mapping = aes(spcount),
colour = "black",
breaks = seq(from = 1000, to = 6000, by = 1)) +
scale_y_continuous(name = "frequency", breaks = c(1, 2, 1))
MR_samples_abun_m %>%
as.data.frame() %>%
select(samplekey, sampleyear, meshsize, preferredspeciesname, spqual, spcount) %>%
filter(spcount >1000) %>%
arrange(preferredspeciesname)
The dataset still contains two sets of taxa name
(speciesname and preferredspeciename) and two
Aphia ID fields (aphiid and preferredaphiaid).
To avoid confusion and aid standardisation we will just keep the
‘preferred’ names and IDs.
MR_samples_abun_m %>%
as.data.frame() %>%
select(speciesname, preferredspeciesname, aphiaid, preferredaphiaid) %>%
filter(is.na(speciesname) | is.na(preferredspeciesname)) %>%
group_by(speciesname) %>%
arrange(speciesname)
speciesname and aphiaid fields
and use preferrredspeciesname and
preferredaphiaid as the taxa name and aphia fieldsuncertain field, this doesn’t have any
consistency or use (and causes problems when grouping)MR_samples_abun_m <- MR_samples_abun_m %>%
select(-c(speciesname, aphiaid, uncertain))
# find all rows with duplicates in every field except count or P/A
dups <- MR_samples_abun_m %>%
as.data.frame() %>%
select(-c(spcount, pa, geometry)) %>%
duplicated()
MR_samples_abun_m %>%
as.data.frame() %>%
select(-c(pa, spcount, geometry)) %>%
group_by_all() %>%
tally() %>%
filter(n > 1)
# pull a list of duplicates taxa in the same sample
qc <- MR_samples_abun_m %>%
as.data.frame() %>%
select(c(samplekey, preferredspeciesname, spqual)) %>%
group_by_all() %>%
tally() %>%
filter(n > 1)
# join on samples with duplicates to create a subset
qc_temp <- left_join(qc, MR_samples_abun_m, by = c("samplekey" = "samplekey", "preferredspeciesname" = "preferredspeciesname", "spqual" = "spqual"), suffix=c("",".y"))
# check values for the duplicates
qc_temp %>%
select(samplekey, preferredspeciesname, spqual, pa, spcount) %>%
arrange(preferredspeciesname, samplekey)
qc_temp <- qc_temp %>%
filter(pa != "P")
qc_temp <- qc_temp %>%
group_by(samplekey, preferredspeciesname, spqual) %>%
dplyr::summarise(spcount_sum = sum(spcount), .groups = "drop")
spcount values# join the summed abundance
MR_samples_abun_m_temp <- left_join(MR_samples_abun_m, qc_temp, by = c("samplekey" = "samplekey", "preferredspeciesname" = "preferredspeciesname", "spqual" = "spqual"), suffix=c("",".y"))
# update the count field for those with a sum
MR_samples_abun_m_temp <- MR_samples_abun_m_temp %>%
mutate(spcount = if_else(is.na(spcount_sum), spcount, spcount_sum))
# remove the duplicated rows that had "present" row and count row
# remove the now redundant sum field
MR_samples_abun_m_temp <- MR_samples_abun_m_temp %>%
filter(is.na(pa) | pa == "" | is.na(spcount)) %>%
select(-c(spcount_sum))
# remove duplicates (those with a sum count will now be duplicates)
MR_samples_abun_m <- MR_samples_abun_m_temp %>%
distinct()
# tidy up the temp data
rm(qc, qc_temp, MR_samples_abun_m_temp)
MR_samples_abun_m %>%
as.data.frame() %>%
select(spqual) %>%
filter(spqual != "") %>%
group_by(spqual) %>%
tally() %>%
arrange(spqual)
dead, eggs, and the
spionids recorded as tubesMR_samples_abun_m %>%
as.data.frame() %>%
select(spqual) %>%
filter(grepl("dead|egg|tubes", spqual, ignore.case = TRUE)) %>%
group_by(spqual) %>%
tally() %>%
arrange(spqual)
MR_samples_abun_m <- MR_samples_abun_m %>%
filter(grepl("dead|egg|tubes", spqual, ignore.case = TRUE) == FALSE)
With taxa names now (partially) standardised to MR ‘preferred names’ we can remove the qualifiers and combine the duplicates. This will aggregate any taxa recorded multiple times in the same sample (e.g. juveniles and adults or species types) and also amalgamate taxa split with different recording policies (e.g. “Syllis sp. A”, “Syllis sp. B”) to the same recorded taxa name (Syllis).
MR_samples_abun_m <- MR_samples_abun_m %>%
mutate(spcount = if_else(pa == "P", 1, as.numeric(spcount)))
MR_samples_abun_m <- MR_samples_abun_m %>%
select(-c(spqual, pa))
# pull a list of column names except for the count
fields <- colnames(MR_samples_abun_m)[colnames(MR_samples_abun_m) != 'spcount' & colnames(MR_samples_abun_m) != 'geometry']
MR_samples_count <- MR_samples_abun_m %>%
as.data.frame() %>%
group_by(across(all_of(fields))) %>%
dplyr::summarise(count_n = sum(spcount)) %>%
ungroup()
# Note - geometry needs re-joining
# Take a data frame of the coordinates per sample
coordinates <- MR_samples_abun_m %>%
select(samplekey, surveykey, geometry) %>%
distinct()
# Join to the aggregated sample data
MR_samples_count <- left_join(MR_samples_count, coordinates, by = c("samplekey" = "samplekey", "surveykey" = "surveykey"))
# tidy up
rm(fields, coordinates)
MR_samples_count %>%
as.data.frame() %>%
group_by(samplekey, preferredspeciesname) %>%
dplyr::summarise(n = n()) %>%
filter(n>1)
samplekey samples with different
repid valuesMR_samples_count %>%
as.data.frame() %>%
select(samplekey, repid) %>%
group_by(samplekey,repid) %>%
dplyr::summarise(n = n()) %>%
group_by(samplekey) %>%
tally() %>%
filter(n >1) %>%
arrange(desc(n))
No duplicate samplekey samples with different
repid (samplekey is unique)
Check for replicates by (exact) location and date
MR_samples_count %>%
as.data.frame() %>%
select(samplekey, sampledate, geometry, geartype) %>%
mutate(coords = as.character(geometry)) %>%
group_by(samplekey, sampledate, coords, geartype) %>%
dplyr::summarise(n = n()) %>%
group_by(coords, sampledate, geartype) %>%
tally() %>%
filter(n >1) %>%
arrange(desc(n))
As expected there are some samples that have been collected as
replicates, although the total number is low. However, there are some
erroneous sample locations highlighted by this aggregation
((-7.000006, 50))
(-7.000006, 50)
coordinates
MRCON04000000005)# plot survey with these coords
MR_samples_count %>%
filter(surveykey == "MRCON04000000005") %>%
ggplot() +
geom_sf(mapping = aes(geometry = geometry))
# samples from this survey
MR_samples_count %>%
as.data.frame() %>%
filter(surveykey == "MRCON04000000005") %>%
mutate(coords = as.character(geometry)) %>%
group_by(samplekey, coords) %>%
count() %>%
arrange(desc(coords))
The coordinates for these samples have an error, we will remove them from the dataset
# filter "(-7.000006, 50)" samples
MR_samples_count <- MR_samples_count %>%
filter(!grepl("MRCON040000001D6|MRCON040000001DA|MRCON040000001F0|MRCON040000001F8|MRCON040000001FC|MRCON04000000206|MRCON0400000020C|MRCON04000000218", samplekey))
Total replicate samples:
MR_samples_count %>%
as.data.frame() %>%
select(samplekey, sampledate, geometry, geartype) %>%
mutate(coords = as.character(geometry)) %>%
group_by(samplekey, sampledate, coords, geartype) %>%
dplyr::summarise(n = n()) %>%
group_by(coords, sampledate, geartype) %>%
tally() %>%
filter(n >1) %>%
ungroup() %>%
dplyr::summarise(total = sum(n))
For the assessment all samples (i.e., all replicates individually) will have Margalef calculated, where replicates are present the median of Margalef D will be taken for the final assessment.
# Processed Marine Recorder data
saveRDS(MR_samples_count, paste0("Data/Processed/R_data/", creation_date(), "_MR_samples_count"))
# find all rows with duplicates in every field
dups <- OB_data_hab_cs_m %>%
as.data.frame() %>%
duplicated()
# list of duplicates
OB_data_hab_cs_m %>%
as.data.frame() %>%
filter(dups) %>%
select(samplecode, year, gearname, validnameworms, abund)
ISLAYOWF2012_42)# example of duplicates
OB_data_hab_cs_m %>%
as.data.frame() %>%
select(samplecode:hab_type) %>%
filter(samplecode == "ISLAYOWF2012_42") %>%
arrange(validnameworms)
OB_data_hab_cs_m <- OB_data_hab_cs_m %>%
filter(dups == FALSE)
# tidy up
rm(dups)
# group by abundance value and check
OB_data_hab_cs_m %>%
as.data.frame() %>%
select(abund) %>%
group_by(abund) %>%
dplyr::summarise(n = n(), .groups = "drop") %>%
arrange(desc(n))
abund field contains values with
data witheld (396
records).OB_data_hab_cs_m %>%
as.data.frame() %>%
filter(abund == "data witheld") %>%
group_by(surveyname, samplecode, year) %>%
count() %>%
group_by(surveyname, year) %>%
summarise(samples = sum(n)) %>%
arrange(surveyname)
## `summarise()` has grouped output by 'surveyname'. You can override using the `.groups` argument.
data witheld also contain normal records# take a list of surveys with "data witheld"
surveys_witheld <- OB_data_hab_cs_m %>%
as.data.frame() %>%
filter(abund == "data witheld") %>%
group_by(surveyname) %>%
count() %>%
chuck("surveyname")
# check for different abundance records
OB_data_hab_cs_m %>%
as.data.frame() %>%
filter(surveyname %in% surveys_witheld) %>%
group_by(abund) %>%
count()
# check taxa names
OB_data_hab_cs_m %>%
as.data.frame() %>%
filter(surveyname %in% surveys_witheld) %>%
group_by(validnameworms) %>%
count()
# tidy up temp data
rm(surveys_witheld)
data witheld abundance also have
data with held for taxa names and no normal sample
records.data with held)OB_data_hab_cs_m <- OB_data_hab_cs_m %>%
filter(abund != "data witheld")
abund doesn’t include any characters it can be
converted to number formatabund field includes
fractions/decimal places (e.g. if samples have been sub-sampled and not
rounded to integers)OB_data_hab_cs_m %>%
as.data.frame() %>%
select(abund) %>%
mutate(abund = as.double(abund))%>%
filter(!abund%%1==0)
OB_data_hab_cs_m <- OB_data_hab_cs_m %>%
mutate(abund = as.integer(abund))
spcount
fieldOB_data_hab_cs_m %>%
as.data.frame() %>%
filter(abund <11) %>%
ggplot() +
geom_histogram(
mapping = aes(abund),
colour = "black",
breaks = seq(from = 0, to = 10, by = 1)) +
scale_y_continuous(name = "frequency") +
scale_x_continuous(breaks = c(1:10))
OB_data_hab_cs_m %>%
as.data.frame() %>%
filter(abund >10 & abund <101) %>%
ggplot() +
geom_histogram(
mapping = aes(abund),
colour = "black",
breaks = seq(from = 10, to = 100, by = 1)) +
scale_y_continuous(name = "frequency") +
scale_x_continuous(breaks = seq(10, 100, 10))
OB_data_hab_cs_m %>%
as.data.frame() %>%
filter(abund >100 & abund <1001) %>%
ggplot() +
geom_histogram(
mapping = aes(abund),
colour = "black",
breaks = seq(from = 100, to = 1000, by = 1)) +
scale_y_continuous(name = "frequency") +
scale_x_continuous(breaks = seq(100, 1000, 100))
OB_data_hab_cs_m %>%
as.data.frame() %>%
filter(abund >1000) %>%
ggplot() +
geom_histogram(
mapping = aes(abund),
colour = "black",
breaks = seq(from = 1000, to = 3000, by = 1)) +
scale_y_continuous(name = "frequency", breaks = c(1, 2, 1))
OB_data_hab_cs_m %>%
as.data.frame() %>%
select(surveyname, year, macrosievesizemm, validnameworms, taxonqualifier, abund) %>%
filter(abund >500) %>%
arrange(validnameworms)
All high abundance records are taxa that are not unusual to occur in high numbers
# find all rows with duplicates in every field except count or P/A
dups <- OB_data_hab_cs_m %>%
as.data.frame() %>%
select(-c(abund, geometry)) %>%
duplicated()
# list of duplicates
OB_data_hab_cs_m %>%
as.data.frame() %>%
filter(dups) %>%
select(samplecode, year, gearname, validnameworms, taxonqualifier) %>%
arrange(validnameworms, samplecode)
17 samples with duplicate taxa records (this includes qualifiers, there will be many more duplicates with qualifiers removed)
qc_temp <- OB_data_hab_cs_m %>%
as.data.frame() %>%
group_by(samplecode, samplecode2, date, validnameworms, taxonqualifier) %>%
dplyr::summarise(abund_sum = sum(abund), .groups = "drop")
# join the summed abundance
OB_samples_abun_temp <- left_join(OB_data_hab_cs_m, qc_temp, by = c("samplecode" = "samplecode", "samplecode2" = "samplecode2", "date" = "date", "validnameworms" = "validnameworms", "taxonqualifier" = "taxonqualifier"), suffix=c("",".y"))
# update the count field for those with a sum, remove the sum field
OB_samples_abun_temp <- OB_samples_abun_temp %>%
mutate(abund = if_else(is.na(abund_sum), abund, abund_sum)) %>%
select(-c(abund_sum))
# remove duplicates (those with a sum count will now be duplicates)
OB_samples_abun <- OB_samples_abun_temp %>%
distinct()
# tidy up the temp data
rm(OB_samples_abun_temp, dups)
OB_samples_abun %>%
as.data.frame() %>%
select(taxonqualifier) %>%
filter(taxonqualifier != "") %>%
group_by(taxonqualifier) %>%
tally() %>%
arrange(taxonqualifier)
Qualifiers are relatively standardised, although there are many with
data witheld
data witheld qualifierOB_samples_abun %>%
as.data.frame() %>%
filter(taxonqualifier == "data witheld") %>%
group_by(surveyname, samplecode) %>%
summarise(records = n()) %>%
group_by(surveyname) %>%
tally()
135 samples with data witheld
qualifier
data witheld qualifier
(Croker Carbonate Slabs MPA 2015_MPACRKS2015)OB_samples_abun %>%
as.data.frame() %>%
filter(surveyname == "Croker Carbonate Slabs MPA 2015_MPACRKS2015") %>%
select(samplecode, validnameworms, taxonqualifier, abund) %>%
arrange(validnameworms) %>%
slice_sample(n =100)
There are regular qualifiers amongst these records
(e.g. juvenile) but no “null” qualifiers, suggests
data witheld is likely to represent NA values and we
shouldn’t exclude samples that contain this qualifier.
eggs and seedsOB_samples_abun <- OB_samples_abun %>%
filter(!grepl("eggs|seeds", taxonqualifier, ignore.case = TRUE))
With unwanted qualifier records removed (e.g. records of “eggs”) we can remove the qualifier field and aggregate any taxa recorded in the same sample with different qualifiers (e.g. juveniles and adults with varying recording policies).
OB_samples_abun <- OB_samples_abun %>%
select(-c(taxonqualifier))
# pull a list of column names except for the abundance
fields <- colnames(OB_samples_abun)[colnames(OB_samples_abun) != 'abund' & colnames(OB_samples_abun) != 'geometry']
OB_samples_count <- OB_samples_abun %>%
as.data.frame() %>%
group_by(across(all_of(fields))) %>%
dplyr::summarise(count_n = sum(abund)) %>%
ungroup()
# Note - geometry needs re-joining
# Take a data frame of the coordinates per sample
coordinates <- OB_samples_abun %>%
select(samplecode, surveyname, geometry) %>%
distinct()
# Join to the aggregated sample data
OB_samples_count <- left_join(OB_samples_count, coordinates, by = c("samplecode" = "samplecode", "surveyname" = "surveyname"))
# tidy up
rm(fields, coordinates)
124 rows amalgamated (identical taxa names previously split by qualifiers)
OB_samples_count %>%
as.data.frame() %>%
group_by(samplecode, validnameworms) %>%
dplyr::summarise(n = n()) %>%
filter(n>1)
samplecode samples with different
samplecode2 valuesOB_samples_count %>%
as.data.frame() %>%
select(samplecode, samplecode2) %>%
group_by(samplecode, samplecode2) %>%
dplyr::summarise(n = n()) %>%
group_by(samplecode) %>%
tally() %>%
filter(n >1) %>%
arrange(desc(n))
No duplicate samplecode samples with different
samplecode2 (samplecode is unique)
Check for replicates by location and date
OB_samples_count %>%
as.data.frame() %>%
select(surveyname, samplecode, date, geometry, gearname) %>%
mutate(coords = as.character(geometry)) %>%
group_by(surveyname, samplecode, date, coords, gearname) %>%
dplyr::summarise(n = n()) %>%
group_by(coords, date, surveyname, gearname) %>%
tally() %>%
filter(n >1) %>%
arrange(surveyname, coords, date)
Some instances of replicates, as to be expected.
Save the data for processing in the next script
# Processed OneBenthic data
saveRDS(OB_samples_count, paste0("Data/Processed/R_data/", creation_date(), "_OB_samples_count"))
# find any rows with duplicates in every field
dups <- WFD_data_hab_cs_m %>%
as.data.frame() %>%
duplicated()
# list of duplicates
WFD_data_hab_cs_m %>%
as.data.frame() %>%
filter(dups)
# tidy up
rm(dups)
abundance# unique abundance values
WFD_data_hab_cs_m %>%
as.data.frame() %>%
select(abundance) %>%
unique() %>%
arrange(abundance)
# check there is no 'na' abundance
WFD_data_hab_cs_m %>%
as.data.frame() %>%
filter(is.na(abundance))
All abundance records are integers or “P” for present
check the distribution of count records in the abundance
field for any unusual frequencies/values
WFD_data_hab_cs_m %>%
as.data.frame() %>%
filter(abundance != "P") %>%
mutate(abundance = as.numeric(abundance)) %>%
ggplot() +
geom_histogram(
mapping = aes(abundance),
colour = "black",
breaks = seq(from = 0, to = 30, by = 1)) +
scale_y_continuous(name = "frequency") +
scale_x_continuous(breaks = c(1:30))
WFD_data_hab_cs_m %>%
as.data.frame() %>%
filter(abundance != "P") %>%
mutate(abundance = as.numeric(abundance)) %>%
ggplot() +
geom_histogram(
mapping = aes(abundance),
colour = "black",
breaks = seq(from = 30, to =150, by = 1)) +
scale_y_continuous(name = "frequency", breaks = seq(0, 16, 2)) +
scale_x_continuous(breaks = seq(30, 150, 10))
WFD_data_hab_cs_m %>%
as.data.frame() %>%
filter(abundance != "P") %>%
mutate(abundance = as.numeric(abundance)) %>%
ggplot() +
geom_histogram(
mapping = aes(abundance),
colour = "black",
breaks = seq(from = 150, to = 800, by = 1)) +
scale_y_continuous(name = "frequency", breaks = seq(0, 3, 1)) +
scale_x_continuous(breaks = seq(150, 800, 50))
As expected, frequency of abundance values rapidly declines with higher values
WFD_data_hab_cs_m %>%
as.data.frame() %>%
filter(abundance != "P") %>%
mutate(abundance = as.numeric(abundance)) %>%
select(surveyname, samplecode, samplemethod, preferredtaxonname, qualifier, abundance) %>%
filter(abundance > 250) %>%
arrange(preferredtaxonname)
No unusual abundance records (Carrick Roads samples are notoriously abundant!)
The EA WFD data includes recorded taxa name (taxonname)
and NHM nomenclature preferred names
(preferredtaxonname)
WFD_data_hab_cs_m %>%
as.data.frame() %>%
select(taxonname, preferredtaxonname) %>%
filter(taxonname != preferredtaxonname) %>%
unique() %>%
arrange(taxonname)
preferredtaxonname are most up to date
taxonname and keep
preferredtaxonname for matching with WoRMS laterWFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
select(-c(taxonname))
# find any rows with duplicates in every field except count or geometry
dups <- WFD_data_hab_cs_m %>%
as.data.frame() %>%
select(-c(abundance, geometry)) %>%
duplicated()
# list duplicates
WFD_data_hab_cs_m %>%
as.data.frame() %>%
filter(dups) %>%
select(samplecode, sampleyear, samplemethod, preferredtaxonname, qualifier) %>%
arrange(preferredtaxonname, samplecode)
Only 2 duplicate records
abundance still includes characters and there are
only two records we will do this individually for each record# update duplicate abundance and remove
WFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
mutate(abundance = if_else(samplecode == "CRT19S001B" & preferredtaxonname == "Copepoda" & abundance == "27", "28", abundance)) %>%
mutate(abundance = if_else(samplecode == "CRT19S003B" & preferredtaxonname == "Copepoda" & abundance == "16", "17", abundance)) %>%
filter(!dups)
# tidy up
rm(dups)
First we will check what qualifiers are present
WFD_data_hab_cs_m %>%
as.data.frame() %>%
select(qualifier) %>%
filter(qualifier != "") %>%
group_by(qualifier) %>%
tally() %>%
arrange(qualifier)
eggsWFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
filter(!grepl("eggs", qualifier, ignore.case = TRUE))
We can now remove the qualifier field and aggregate any taxon recorded in a sample with different qualifiers
quailiferWFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
select(-c(qualifier))
# find "P" records
WFD_presence <- WFD_data_hab_cs_m %>%
as.data.frame() %>%
filter(abundance == "P") %>%
select(samplecode, preferredtaxonname, abundance)
# find count records
WFD_count <- WFD_data_hab_cs_m %>%
as.data.frame() %>%
filter(abundance != "P") %>%
select(samplecode, preferredtaxonname, abundance)
# find those that have both count and P records
WFD_presence_dup <- inner_join(WFD_presence, WFD_count, by = c("samplecode" = "samplecode", "preferredtaxonname" = "preferredtaxonname"), suffix = c("", ".y")) %>%
print()
## samplecode preferredtaxonname abundance abundance.y
## 1 CRT19S010B Spionidae P 2
## 2 CRT19S005B Spionidae P 56
## 3 CRT19S012B Spionidae P 22
# filter by join
WFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
anti_join(WFD_presence_dup, by = c("samplecode" = "samplecode", "preferredtaxonname" = "preferredtaxonname", "abundance" = "abundance"))
# tidy up
rm(WFD_presence, WFD_count, WFD_presence_dup)
Now there are no duplicate presence and count records for the same taxa in a sample we can convert all “P” records to 1. At later stages taxonomic recording policy (presence/absence or count) will be further standradised in line with the BH2b CEMP guidelines.
P to 1
abundance to a numeric fieldWFD_data_hab_cs_m <- WFD_data_hab_cs_m %>%
mutate(abundance = if_else(abundance == "P", "1", abundance),
abundance = as.numeric(abundance))
Now all abundance values are numeric we can aggregate duplicate taxa in samples (previously with different quailifiers)
# pull a list of column names except for the abundance
fields <- colnames(WFD_data_hab_cs_m)[colnames(WFD_data_hab_cs_m) != 'abundance' & colnames(WFD_data_hab_cs_m) != 'geometry']
WFD_data_hab_cs_count <- WFD_data_hab_cs_m %>%
as.data.frame() %>%
group_by(across(all_of(fields))) %>%
dplyr::summarise(count_n = sum(abundance)) %>%
ungroup()
# Note - geometry needs re-joining
# Take a data frame of the coordinates per sample
coordinates <- WFD_data_hab_cs_m %>%
select(samplecode, surveyname, geometry) %>%
distinct()
# Join to the aggregated sample data
WFD_samples_count <- left_join(WFD_data_hab_cs_count, coordinates, by = c("samplecode" = "samplecode", "surveyname" = "surveyname"))
# tidy up
rm(fields, coordinates)
36 rows amalgamated (identical taxa names previously split by qualifiers)
WFD_samples_count %>%
as.data.frame() %>%
group_by(samplecode, preferredtaxonname) %>%
dplyr::summarise(n = n()) %>%
filter(n>1)
WFD_samples_count %>%
as.data.frame() %>%
select(surveyname, samplecode, sampledate, geometry, samplemethod) %>%
mutate(coords = as.character(geometry)) %>%
group_by(surveyname, samplecode, sampledate, coords, samplemethod) %>%
dplyr::summarise(n = n()) %>%
group_by(coords, sampledate, surveyname, samplemethod) %>%
tally() %>%
filter(n >1) %>%
arrange(surveyname, coords, sampledate)
No duplicate samples taken with exactly the same location, date and gear type
Save the EA WFD data for processing in the next script
# Processed WFD data
saveRDS(WFD_samples_count, paste0("Data/Processed/R_data/", creation_date(), "_WFD_samples_count"))
# find any rows with duplicates in every field
dups <- merman_data_hab_cs %>%
as.data.frame() %>%
duplicated()
# list of duplicates
merman_data_hab_cs %>%
as.data.frame() %>%
filter(dups)
# tidy up
rm(dups)
abundance and
result# unique abundance values
merman_data_hab_cs %>%
as.data.frame() %>%
select(abundance) %>%
unique() %>%
arrange(abundance)
merman_data_hab_cs %>%
as.data.frame() %>%
select(result) %>%
unique() %>%
arrange(result)
# check there is no 'na' abundance
merman_data_hab_cs %>%
as.data.frame() %>%
filter(!is.na(result))
abundance contains records of ‘presence’,
result includes numerical counts
results recordsmerman_data_hab_cs %>%
as.data.frame() %>%
filter(result == "#MULTIVALUE"| result == "0") %>%
arrange(result)
#MULTIVALUE is only recorded for COPEPODA, we’ll filter
these records out as records of COPEPODA will not be retained in the
final analysis
merman_data_hab_cs <- merman_data_hab_cs %>%
filter(result != "#MULTIVALUE")
0 result counts are recorded with
Presence
Presencemerman_data_hab_cs %>%
as.data.frame() %>%
filter(abundance == "Presence") %>%
arrange(desc(result))
No other counts with Presence records
0s to NAmerman_data_hab_cs <- merman_data_hab_cs %>%
mutate(result = na_if(result, "0"))
# find all rows with duplicates in every field except results or abundance
dups <- merman_data_hab_cs %>%
as.data.frame() %>%
select(-c(result, abundance, geometry)) %>%
duplicated()
# find the fields to filter the data
dup_list <- merman_data_hab_cs %>%
as.data.frame() %>%
filter(dups) %>%
select(sampleref, samplestartdate, specieslatinname)
# Show duplicates
merman_data_hab_cs %>%
as.data.frame() %>%
right_join(dup_list)
Whilst there is no duplicates with count records and
Presence records, there are duplicate counts for
Clymenura in these five samples
result doesn’t contain decimalsresult to numericalPresence recordsabundancefield# check for decimals
merman_data_hab_cs %>%
as.data.frame() %>%
mutate(result = as.double(result)) %>%
filter(!result%%1==0)
## Warning in `[<-.data.frame`(`*tmp*`, is_list, value = list(`21` =
## "<s_GEOMET>")): replacement element 1 has 1 row to replace 0 rows
# change to numerical type
merman_data_hab_cs <- merman_data_hab_cs %>%
mutate(result = as.double(result))
# update Presence to 1
merman_data_hab_cs <- merman_data_hab_cs %>%
mutate(result = if_else(abundance == "Presence", 1, result))
# remove "Presence" field
merman_data_hab_cs <- merman_data_hab_cs %>%
select(-c(abundance))
# sum duplicate records
qc_temp <- merman_data_hab_cs %>%
as.data.frame() %>%
right_join(dup_list) %>%
mutate(result = as.double(result)) %>%
group_by(sampleref, samplestartdate, specieslatinname) %>%
summarise(count_sum = sum(result))
# add sum values and remove duplicates
merman_data_hab_cs <- merman_data_hab_cs %>%
left_join(qc_temp) %>%
filter(!dups) %>%
mutate(result = if_else(is.na(count_sum), result, count_sum)) %>%
select(-c(count_sum))
# tidy up
rm(dups, dup_list, qc_temp)
check the distribution of count records in the `result`` field for any unusual frequencies/values
merman_data_hab_cs %>%
as.data.frame() %>%
ggplot() +
geom_histogram(
mapping = aes(result),
colour = "black",
breaks = seq(from = 0, to = 30, by = 1)) +
scale_y_continuous(name = "frequency") +
scale_x_continuous(breaks = c(1:30))
merman_data_hab_cs %>%
as.data.frame() %>%
ggplot() +
geom_histogram(
mapping = aes(result),
colour = "black",
breaks = seq(from = 30, to =150, by = 1)) +
scale_y_continuous(name = "frequency", breaks = seq(0, 60, 5)) +
scale_x_continuous(breaks = seq(30, 150, 10))
merman_data_hab_cs %>%
as.data.frame() %>%
ggplot() +
geom_histogram(
mapping = aes(result),
colour = "black",
breaks = seq(from = 150, to = 2600, by = 1)) +
scale_y_continuous(name = "frequency", breaks = seq(0, 5, 1)) +
scale_x_continuous(breaks = seq(100, 2600, 200))
As expected, frequency of abundance values rapidly declines with higher values
merman_data_hab_cs %>%
as.data.frame() %>%
select(sampleref, monitoringyear, specieslatinname, stageofdevelopment, result, msfd) %>%
filter(result > 250) %>%
arrange(specieslatinname)
Although some counts are high, none appear to be erroneous. The record of 2500 Gobiidae eggs will be removed in the next stages.
# 1st qualifier
merman_data_hab_cs %>%
as.data.frame() %>%
group_by(stageofdevelopment) %>%
count(stageofdevelopment)
# 2nd qualifier
merman_data_hab_cs %>%
as.data.frame() %>%
group_by(sexofspecies) %>%
count(sexofspecies)
Egg qualifier# remove eggs
merman_data_hab_cs <- merman_data_hab_cs %>%
filter(grepl("Egg", stageofdevelopment) == FALSE)
dups <- merman_data_hab_cs %>%
as.data.frame() %>%
select(-c(stageofdevelopment, sexofspecies)) %>%
duplicated()
# find the fields to filter the data
dup_list <- merman_data_hab_cs %>%
as.data.frame() %>%
filter(dups) %>%
select(sampleref, samplestartdate, specieslatinname)
# Show duplicates
merman_data_hab_cs %>%
as.data.frame() %>%
right_join(dup_list)
## Joining, by = c("sampleref", "samplestartdate", "specieslatinname")
stageofdevelopment &
sexofspecies)# sum duplicate records
qc_temp <- merman_data_hab_cs %>%
as.data.frame() %>%
right_join(dup_list) %>%
mutate(result = as.double(result)) %>%
group_by(sampleref, samplestartdate, specieslatinname) %>%
summarise(count_sum = sum(result))
# add sum values and remove duplicates
merman_data_hab_cs <- merman_data_hab_cs %>%
left_join(qc_temp) %>%
filter(!dups) %>%
mutate(result = if_else(is.na(count_sum), result, count_sum)) %>%
select(-c(count_sum, stageofdevelopment, sexofspecies))
# tidy up
rm(dups, dup_list, qc_temp)
sampleref is not a unique sample identifier
(combination of station name and replicate name)samplecode field (combination of
sampleref and samplestartdate)merman_samples_count <- merman_data_hab_cs %>%
as.data.frame() %>%
mutate(samplecode = paste(sampleref, samplestartdate, sep = "_"))
Save the MERMAN data for processing in the next script
# Processed MERMAN data
saveRDS(merman_samples_count, paste0("Data/Processed/R_data/", creation_date(), "_merman_samples_count"))
Visualise the gear types and habitats across years in the different datasets following initial filtering and cleaning.
# take a summary of MR samples
MR_temp <- MR_samples_count %>%
as.data.frame() %>%
group_by(samplekey, sampledate, sampleyear, geartype, msfd) %>%
dplyr::summarise(n = n(), .groups = "drop") %>%
mutate(dataset = "MR")
# take a summary of OB samples
OB_temp <- OB_samples_count %>%
as.data.frame() %>%
group_by(samplecode, sampledate, year, gearname, msfd) %>%
dplyr::summarise(n = n(), .groups = "drop") %>%
dplyr::rename(samplekey = samplecode, sampleyear = year, geartype = gearname) %>%
mutate(dataset = "OB")
# take a summary of WFD samples
WFD_temp <- WFD_samples_count %>%
as.data.frame() %>%
group_by(samplecode, sampledate, sampleyear, samplemethod, msfd) %>%
dplyr::summarise(n = n(), .groups = "drop") %>%
dplyr::rename(samplekey = samplecode, geartype = samplemethod) %>%
mutate(dataset = "WFD")
# take a summary of MERMAN samples
merman_temp <- merman_samples_count %>%
as.data.frame() %>%
group_by(samplecode, samplestartdate, monitoringyear, samplertype, msfd) %>%
dplyr::summarise(n = n(), .groups = "drop") %>%
dplyr::rename(samplekey = samplecode, sampleyear = monitoringyear, sampledate = samplestartdate, geartype = samplertype) %>%
mutate(dataset = "MERMAN",
sampledate = lubridate::dmy(sampledate))
# take a summary of NRW samples (samples were mostly Day grabs)
NRW_temp <- samples_NRW_bht %>%
select(sampleref, sampledate, sampleyear, msfd) %>%
mutate(geartype = "Day") %>%
dplyr::rename(samplekey = sampleref) %>%
mutate(dataset = "NRW")
# combine sample summaries
combined_temp <- bind_rows(MR_temp, OB_temp, WFD_temp, merman_temp, NRW_temp)
# tidy up
rm(MR_temp, OB_temp, WFD_temp, merman_temp, NRW_temp)
# standardise gear type names
combined_temp <- combined_temp %>%
as.data.frame() %>%
mutate(geartype = case_when(grepl("day", geartype, ignore.case = T) ~ "Day Grab",
grepl("hamon", geartype, ignore.case = T) ~ "Hamon Grab",
grepl("Veen", geartype, ignore.case = T) ~ "Van Veen Grab",
TRUE ~ geartype))
# pull a list of gear names
gearlist <- combined_temp %>%
chuck("geartype") %>%
unique %>%
sort
# summary of sample methods and years
combined_temp %>%
ggplot() +
geom_jitter(
mapping = aes(x = sampledate, y = match(geartype, gearlist), colour = dataset),
alpha = 0.5, width = 0, height = 0.2) +
scale_x_date(name = "", date_breaks = "1 years", date_labels = "%Y") +
scale_y_continuous(name = "", breaks = seq_len(length(gearlist)), labels = gearlist) +
scale_color_manual(values = cb_palette) +
labs(title = "Gear Types") +
theme(legend.title=element_blank()) +
theme_bw() +
guides(color=guide_legend(title="Dataset", override.aes = list(size=3)))
# clean up
rm(gearlist)
hablist <- combined_temp %>%
chuck("msfd") %>%
unique %>%
sort(decreasing = TRUE) %>%
str_replace("and/or", "and/or\n")
# summary of samples and years
combined_temp %>%
mutate(msfd = str_replace(msfd, "and/or", "and/or\n")) %>%
ggplot() +
geom_jitter(
mapping = aes(x = sampledate, y = match(msfd, hablist), colour = dataset),
alpha = 0.5, width = 0, height = 0.2) +
scale_color_manual(values = cb_palette) +
scale_x_date(name = "", date_breaks = "1 years", date_labels = "%Y") +
scale_y_continuous(name = "", breaks = seq_len(length(hablist)), labels = hablist) +
labs(title = "Broad-scale Habitats Sampled") +
theme(legend.title=element_blank()) +
theme_bw() +
guides(color=guide_legend(title="Dataset", override.aes = list(size=3)))
# tidy up
rm(hablist)
Data preparation and analysis is continued in the next script: Data Truncation
details::details(sessionInfo(), summary = 'sessionInfo')
R version 3.6.1 (2019-07-05)
Platform: x86_64-w64-mingw32/x64 (64-bit)
Running under: Windows Server x64 (build 14393)
Matrix products: default
locale:
[1] LC_COLLATE=English_United Kingdom.1252
[2] LC_CTYPE=English_United Kingdom.1252
[3] LC_MONETARY=English_United Kingdom.1252
[4] LC_NUMERIC=C
[5] LC_TIME=English_United Kingdom.1252
attached base packages:
[1] stats graphics grDevices utils datasets methods base
other attached packages:
[1] RColorBrewer_1.1-2 details_0.3.0 DT_0.26 sf_0.9-8
[5] forcats_0.5.1 stringr_1.4.0 dplyr_1.0.7 purrr_0.3.4
[9] readr_2.1.0 tidyr_1.1.4 tibble_3.1.6 ggplot2_3.3.5
[13] tidyverse_1.3.1 rprojroot_2.0.3
loaded via a namespace (and not attached):
[1] httr_1.4.2 sass_0.4.4 jsonlite_1.7.2 modelr_0.1.8
[5] bslib_0.4.1 assertthat_0.2.1 highr_0.9 cellranger_1.1.0
[9] yaml_2.2.1 pillar_1.6.4 backports_1.3.0 glue_1.5.0
[13] digest_0.6.28 rvest_1.0.2 colorspace_2.0-2 htmltools_0.5.2
[17] clipr_0.7.1 pkgconfig_2.0.3 broom_0.7.10 haven_2.4.3
[21] s2_1.0.4 scales_1.1.1 tzdb_0.2.0 proxy_0.4-27
[25] generics_0.1.1 farver_2.1.0 ellipsis_0.3.2 cachem_1.0.6
[29] withr_2.5.0 cli_3.1.0 magrittr_2.0.1 crayon_1.4.2
[33] readxl_1.3.1 evaluate_0.18 fs_1.5.0 fansi_0.5.0
[37] xml2_1.3.2 class_7.3-15 tools_3.6.1 hms_1.1.1
[41] lifecycle_1.0.1 munsell_0.5.0 reprex_2.0.1 compiler_3.6.1
[45] jquerylib_0.1.4 e1071_1.7-11 rlang_0.4.12 classInt_0.4-3
[49] units_0.8-0 grid_3.6.1 rstudioapi_0.13 htmlwidgets_1.5.4
[53] crosstalk_1.2.0 labeling_0.4.2 rmarkdown_2.18 wk_0.6.0
[57] gtable_0.3.0 DBI_1.1.3 R6_2.5.1 lubridate_1.8.0
[61] knitr_1.41 fastmap_1.1.0 utf8_1.2.2 KernSmooth_2.23-15
[65] desc_1.4.2 stringi_1.7.5 Rcpp_1.0.10 vctrs_0.3.8
[69] png_0.1-7 dbplyr_2.1.1 tidyselect_1.1.1 xfun_0.35