Added some comments, variable for ngramfile, cleanup feature ranking aggregation in 10fold.
diff --git a/R/idiomclassification_mk_pf.R b/R/idiomclassification_mk_pf.R
index 09aee79..4880c9f 100644
--- a/R/idiomclassification_mk_pf.R
+++ b/R/idiomclassification_mk_pf.R
@@ -6,9 +6,11 @@
# library(randomForestExplainer)
# may need to: options(expressions = 5e5) to avoid stackoverflow for installing package
+ngramfile<-"gold03_anno_ml_synfeat_nstopw"
+
setwd(dirname(rstudioapi::getSourceEditorContext()$path))
stopwords <- readLines(con = "../data/stopwords.txt",encoding="UTF-8")
-oringramme <- read.csv("../data/gold03_anno_ml_synfeat_nstopw.csv", header = TRUE, sep = "\t", dec=".", quote="", encoding="UTF-8",stringsAsFactors=FALSE)
+oringramme <- read.csv(paste("../data/",ngramfile,".csv",sep=""), header = TRUE, sep = "\t", dec=".", quote="", encoding="UTF-8",stringsAsFactors=FALSE)
syfeaturenames <- read.csv("../data/syfeatures.tsv", header = TRUE, sep = "\t", dec=".", quote="", encoding="UTF-8",stringsAsFactors=FALSE)
# syfeaturenames$navalue<-sapply(syfeaturenames$navalue,as.numeric)
@@ -19,6 +21,8 @@
oringramme <- oringramme %>%
filter(CO_IDIOM < 2) # just two classes: 0 no idiom, 1 idiom
+# Reduce number of classes, treat null values, add NSTOPW, change names for SY features
+
ngramme <- oringramme %>%
add_column(NSTOPW = sapply(oringramme$tokens,function(x) length(deleteStopwords(tolower(unlist(strsplit(x," "))),stopwords)))) %>%
# select(-matches("CO_TOKEN.*"), -tokens) %>%
@@ -27,12 +31,19 @@
mutate(across(c("dice", "lfmd", "llr", "ld", "pmi"), ~ replace_na(.x, min(.x) - 1))) %>%
rename_at(syfeaturenames$innames, ~ syfeaturenames[syfeaturenames$innames==.x,]$synames ) %>%
mutate(across(everything(), ~ replace_na(.x, 0))) %>%
- mutate(CO_IDIOM = as.factor(if_else(CO_IDIOM !=1, "0", "1")))
+ mutate(CO_IDIOM = as.factor(if_else(CO_IDIOM !=1, "0", "1"))) # just two classes: 0 no idiom, 1 idiom
+
+# Optional
+write.table(ngramme,file=paste("../data/",ngramfile,"_cosy.csv",sep=""), sep = "\t", quote=F)
+
+# featuresets
covars <- c("CO_LL", "CO_Z", "CO_G", "CO_T", "CO_LOGDICE", "CO_PMI", "CO_MI3", "CO_DEREKO", "CO_SGT", "CO_WIN5_VEC","CO_WIN5_VEC_AUTOSEM")
syvars <- c(syfeaturenames$synames,"NSTOPW")
vars <- c(covars,syvars)
+# formulae for training and testing rf
+
fmla <- as.formula(paste("CO_IDIOM ~ ", paste(vars, collapse= "+")))
fmlaco <- as.formula(paste("CO_IDIOM ~ ", paste(covars, collapse= "+")))
fmlasy <- as.formula(paste("CO_IDIOM ~ ", paste(syvars, collapse= "+")))
@@ -53,23 +64,28 @@
# different cutoff for prediction
# prediction_for_table <- predict(rf_classifier, test %>% select(-CO_IDIOM), cutoff = c(0.8, 0.2))
-confusion <- table(predicted=prediction_for_table, observed=test$CO_IDIOM)
-conf <- confusionMatrix(confusion, positive = "1")
+confusion <- table(observed=test$CO_IDIOM,predicted=prediction_for_table)
+conf <- confusionMatrix(confusion, positive= "1")
print(conf)
+# Sensitivity is precision of class 1
+# Pos Pred Value is recall
varImpPlot(rf_classifier)
# optional resampling with smote
smoted.data <- SMOTE(fmla, subset(train, select = c("CO_IDIOM", vars)), perc.over = 1200, perc.under = 100)
-rf_classifier = randomForest(fmla, smoted.data, ntree=200, importance=TRUE)
-prediction_for_table <- predict(rf_classifier,test %>% select(-CO_IDIOM), cutoff=c(0.8,0.2))
-confusion <- table(predicted=prediction_for_table, observed=test$CO_IDIOM)
+rf_classifier = randomForest(fmla, smoted.data, ntree=100, mtry=10, importance=TRUE)
+prediction_for_table <- predict(rf_classifier,test %>% select(-CO_IDIOM))
+confusion <- table(observed=test$CO_IDIOM,predicted=prediction_for_table)
conf <- confusionMatrix(confusion, positive = "1")
print(conf)
+
# Using estimates by random forest on entire dataset
+
library(randomForest)
rf_classifier_full = randomForest(fmla, data=ngramme, ntree=100, mtry=2, importance=TRUE, cutoff=c(0.8,0.2))
rf_classifier_full
+# class.error is 1 - recall
varImpPlot(rf_classifier_full)
# Feature ranking
@@ -98,19 +114,20 @@
#randomForestExplainer::explain_forest(rf_classifier )
# averate estimates and feature ranks over 10 runs
+
errrate<-0
conf<-matrix(0,2,3)
featureRanks<-matrix(0,4,length(vars))
for (i in 1:10) {
- # rfc =randomForest(fmla, data=ngramme, ntree=100, importance=TRUE)
- rfc =randomForest(fmla, data=ngramme, ntree=100, importance=TRUE, cutoff=c(0.8,0.2))
+ rfc =randomForest(fmla, data=ngramme, ntree=100, importance=TRUE)
+ #rfc =randomForest(fmla, data=ngramme, ntree=100, importance=TRUE, cutoff=c(0.8,0.2))
errrate<-errrate+rfc$err.rate[100,1]
conf<-conf+rfc$confusion
featureRanks<-featureRanks+
cbind(importance(rfc)[,3:4],
+ information.gain(fmla, data=ngramme%>%mutate_at(vars, ~ . * 1000),unit="log2"),
sapply(vars,
- function(sel) t.test(idioms[sel],nonidioms[sel])$p.value),
- information.gain(fmla, data=ngramme%>%mutate_at(vars, ~ . * 1000),unit="log2"))
+ function(sel) t.test(idioms[sel],nonidioms[sel])$p.value))
print(errrate/i)
conf1<-round(
rbind(
@@ -122,6 +139,7 @@
print(conf1)
}
featureRanks<-featureRanks/10
+colnames(featureRanks)<-c("MeanDecreaseAccuracy","MeanDecreaseGini","InformationGain","Ttest")