Reddit Comments Analysis

<<JLink`;
InstallJava[];
ReinstallJava[JVMArguments->"-Xmx32000m"]

Needs["DatabaseLink`"]

conn=OpenSQLConnection[]

SQLTables[conn]

SQLColumns[conn,"May2015"]

UnixTime[{2015,5,19,2,46,0}] (*Mercury Retrograde Starts*)

(*timesandwords=SQLSelect[ conn,"May2015",{"created_utc","body"}]*)

NMR=SQLSelect[ conn,"May2015",{"created_utc","body"},SQLColumn["created_utc"]<1432003560]

Length[NMR]

(*selectionsNMR=RandomInteger[{1,Length[NMR]},2000000]*)

(*ClearAll[a,i,j,misspell];
(*timeNMR={};*)
text={};
fractionNMR={};
text={};
textwords={};
stringslist={};
For[a=1,a≤1000(*Length[selectionsNMR]*),a++, (*a is the entry number*)
misspell=0;
text=NMR(*[[selectionsNMR[[a]]]]*)[[a]][[2]];
(*CHECK FOR ENGLISH*)
If [LanguageIdentify[text]==Entity["Language","English"],
text=StringReplace[text,"1"->""]; (*Remove numbers*)
text=StringReplace[text,"2"->""];
text=StringReplace[text,"3"->""];
text=StringReplace[text,"4"->""];
text=StringReplace[text,"5"->""];
text=StringReplace[text,"6"->""];
text=StringReplace[text,"7"->""];
text=StringReplace[text,"8"->""];
text=StringReplace[text,"9"->""];
text=StringReplace[text,"0"->""];
text=StringReplace[text,"'s"->""];
text=StringReplace[text,"\\n"->" "]; (*Remove paragraph breaks*)
textwords=TextWords[text];
For [j=1,j<=Length[textwords],j++, (*j is the particular word number*)
wordstring=ToString[textwords[[j]]];
If [Not[SubsetQ[Characters[wordstring],{"."}]]&&Not[SubsetQ[Characters[wordstring],{":"}]]&&Not[wordstring]=="ok"&&Not[wordstring=="NSFL"]&& Not[wordstring=="NSFW"],(*checks for links and some slang and most emojis*)
If[DictionaryLookup[wordstring,IgnoreCase->True]=={},(* gets rid of capitalization and checks for spelling*)
(*If [StringDrop[DictionaryLookup[wordstring,IgnoreCase->True],1][[1]]≠StringDrop[ToString[textwords[[j]]],1],misspell=misspell+1],*)If[StringLength[wordstring]≥4,If[Not[StringTake[wordstring,4]=="fuck"]&&Not[StringTake[wordstring,4]=="shit"],misspell=misspell+1],misspell=misspell+1](*hell and damn ARE in dictionary*)
If[StringLength[wordstring]<4, misspell=misspell+1];
];
];
];
If [Length[textwords]≠0&&N[misspell/j]≠1.0,AppendTo[fractionNMR,N[misspell/j]]];
];
];*)

MR=SQLSelect[ conn,"May2015",{"created_utc","body"},SQLColumn["created_utc"]>=1432003560];

CloseSQLConnection[conn]

Length[MR]

ClearAll[a,i,j,misspell];
fractionsNMR={};
lengthsNMR={};
fractionsMR={};
lengthsMR={};
For[i=1,i<31,i++,
selectionsNMR=RandomInteger[{1,Length[NMR]},100000];
selectionsMR=RandomInteger[{1,Length[MR]},100000];(*RandomInteger[{1,Length[MR]},Round[N[(100000/Length[NMR])*Length[MR]]]];*)
text={};
fractionNMR={};
lengthNMR={};
fractionMR={};
lengthMR={};
text={};
textwords={};
stringslist={};
For[a=1,a≤100000(*Length[selectionsNMR]*),a++, (*a is the entry number*)
misspell=0;
text=NMR[[selectionsNMR[[a]]]](*[[a]]*)[[2]];
(*CHECK FOR ENGLISH*)
If [LanguageIdentify[text]==Entity["Language","English"],
text=StringReplace[text,"1"->""]; (*Remove numbers*)
text=StringReplace[text,"2"->""];
text=StringReplace[text,"3"->""];
text=StringReplace[text,"4"->""];
text=StringReplace[text,"5"->""];
text=StringReplace[text,"6"->""];
text=StringReplace[text,"7"->""];
text=StringReplace[text,"8"->""];
text=StringReplace[text,"9"->""];
text=StringReplace[text,"0"->""];
text=StringReplace[text,"'s"->""];
text=StringReplace[text,"\n"->" "];
text=StringReplace[text,"\\n"->" "]; (*Remove paragraph breaks*)
textwords=TextWords[text];
For [j=1,j<=Length[textwords],j++, (*j is the particular word number*)
wordstring=(*ToString[*)textwords[[j]](*]*);
If [Not[SubsetQ[Characters[wordstring],{"."}]]&&Not[SubsetQ[Characters[wordstring],{":"}]]&&Not[wordstring=="bs"]&&Not[wordstring=="BS"]&&Not[wordstring=="ok"]&&Not[wordstring=="OK"]&&Not[wordstring=="gg"]&&Not[wordstring=="GG"]&&Not[wordstring=="awol"]&&Not[wordstring=="AWOL"]&&Not[wordstring=="4chan"]&&Not[wordstring=="4Chan"]&& Not[wordstring=="NSFW"]&&Not[wordstring=="nsfw"]&& Not[wordstring=="NSFL"]&&Not[wordstring=="nsfl"]&&Not[wordstring=="ama"]&&Not[wordstring=="AMA"]&&Not[wordstring=="lol"]&&Not[wordstring=="LOL"]&&Not[SubsetQ[Characters[wordstring],{";"}]],(*checks for links and some slang and most emojis*)
If[DictionaryLookup[wordstring,IgnoreCase->True]=={},(* gets rid of capitalization and checks for spelling*)
If[StringLength[wordstring]≥4,If[Not[StringTake[wordstring,4]=="fuck"]&&Not[StringTake[wordstring,4]=="shit"],misspell=misspell+1];];(*hell and damn ARE in dictionary*)
If[StringLength[wordstring]<4, misspell=misspell+1];
];
];
];
If [Length[textwords]≠0&&N[misspell/Length[textwords]]≠1.0,AppendTo[fractionNMR,N[misspell/Length[textwords]]]];
AppendTo[lengthNMR,Length[textwords]];
];
(*text={};
fractionMR={};
lengthMR={};
text={};
textwords={};
stringslist={};*)(*For[b=1,b≤100000(*Length[selectionsMR]*),b++, (*b is the entry number*)*)
misspell=0;
text=MR[[selectionsMR[[a]]]](*[[a]]*)[[2]];
(*CHECK FOR ENGLISH*)
If [LanguageIdentify[text]==Entity["Language","English"],
text=StringReplace[text,"1"->""]; (*Remove numbers*)
text=StringReplace[text,"2"->""];
text=StringReplace[text,"3"->""];
text=StringReplace[text,"4"->""];
text=StringReplace[text,"5"->""];
text=StringReplace[text,"6"->""];
text=StringReplace[text,"7"->""];
text=StringReplace[text,"8"->""];
text=StringReplace[text,"9"->""];
text=StringReplace[text,"0"->""];
text=StringReplace[text,"'s"->""];
text=StringReplace[text,"\n"->" "];
text=StringReplace[text,"\\n"->" "]; (*Remove paragraph breaks*)
textwords=TextWords[text];
For [j=1,j<=Length[textwords],j++, (*j is the particular word number*)
wordstring=(*ToString[*)textwords[[j]](*]*);
If [Not[SubsetQ[Characters[wordstring],{"."}]]&&Not[SubsetQ[Characters[wordstring],{":"}]]&&Not[wordstring=="bs"]&&Not[wordstring=="BS"]&&Not[wordstring=="ok"]&&Not[wordstring=="OK"]&&Not[wordstring=="gg"]&&Not[wordstring=="GG"]&&Not[wordstring=="awol"]&&Not[wordstring=="AWOL"]&&Not[wordstring=="4chan"]&&Not[wordstring=="4Chan"]&& Not[wordstring=="NSFW"]&&Not[wordstring=="nsfw"]&& Not[wordstring=="NSFL"]&&Not[wordstring=="nsfl"]&&Not[wordstring=="ama"]&&Not[wordstring=="AMA"]&&Not[wordstring=="lol"]&&Not[wordstring=="LOL"]&&Not[SubsetQ[Characters[wordstring],{";"}]],(*checks for links and some slang and most emojis*) If[DictionaryLookup[wordstring,IgnoreCase->True]=={},(* gets rid of capitalization and checks for spelling*) If[StringLength[wordstring]≥4,If[Not[StringTake[wordstring,4]=="fuck"]&&Not[StringTake[wordstring,4]=="shit"],misspell=misspell+1];];(*hell and damn ARE in dictionary*) If[StringLength[wordstring]<4, misspell=misspell+1];
];
];
];
If [Length[textwords]≠0(*(*&&N[misspell/Length[textwords]]≠1.0*),*)AppendTo[fractionMR,N[misspell/Length[textwords]]]];
AppendTo[lengthMR,Length[textwords]];
];
];
AppendTo[fractionsNMR,N[Mean[fractionNMR]]];
AppendTo[lengthsNMR,N[Mean[lengthNMR]]];
AppendTo[fractionsMR,N[Mean[fractionMR]]];
AppendTo[lengthsMR,N[Mean[lengthMR]]];
];

Not["ama"=="awol"||"bs"]

TextWords[";) hello"]

Mean[fractionNMR]

Mean[fractionMR]

N[Dot[fractionsMR[[1;;i-1]],(*.*)(1/fractionsNMR[[1;;i-1]])]*1/(i-1)]

N[Dot[lengthsMR[[1;;i-1]],(*.*)(1/lengthsNMR[[1;;i-1]])]*1/(i-1)]

N[Mean[lengthsMR]/Mean[lengthsNMR]]

N[Mean[fractionsMR]/Mean[fractionsNMR]]

N[Mean[fractionMR]/Mean[fractionNMR]]

N[Mean[lengthMR]/Mean[lengthNMR]]

a

i

fractionsMR

fractionsNMR

N[Mean[lengthsNMR]]

Variance[fractionsNMR]

Variance[fractionsMR]

Export["fractionsMR.dat",fractionsMR]

Export["fractionsNMR.dat",fractionsNMR]

Export["lengthsNMR.dat",lengthsNMR]

Export["lengthsMR.dat",lengthsMR]

Needs["HypothesisTesting`"]

MeanCI[fractionsNMR]

MeanCI[fractionsMR]

Histogram[{fractionsMR,fractionsNMR},ChartLegends->{"MR","NMR"}]

Histogram[{lengthsMR,lengthsNMR},ChartLegends->{"MR","NMR"}]

MeanCI[lengthsNMR]

MeanCI[lengthsMR]

Exp[0]

fractionsNMR

QuantilePlot[fractionsMR,fractionsNMR,PlotLegends->{"NMR","MR"}]

ProbabilityPlot[fractionsMR,fractionsNMR]

QuantilePlot[lengthsMR,lengthsNMR,PlotLegends->{"NMR","MR"}]

ProbabilityPlot[lengthsMR,lengthsNMR]

LocationEquivalenceTest[{fractionsMR,fractionsNMR},{"TestDataTable",All}]

LocationEquivalenceTest[{lengthsMR,lengthsNMR},{"TestDataTable",All}]

i

fractionsMR

Table[Mean[N[fractionsMR[[m]]*(1/fractionsNMR[[1;;i-1]])]],{m,i-1}]

Length[%95]

Length[%64]

Mean[Mean[Table[N[lengthsMR[[m]]*(1/lengthsNMR[[1;;i-1]])*1/(i-1)],{m,i-1}]]*30]

Table[N[Dot[lengthsMR[[1;;m]],(*.*)(1/lengthsNMR[[1;;i-1]])]*1/(i-1)],{m,i-1}]

{DistributionFitTest[fractionsNMR],DistributionFitTest[fractionsNMR],DistributionFitTest[lengthsNMR],DistributionFitTest[lengthsNMR]}

Focusing on simple emoji:

ClearAll[a,i,j,emoji];
efractionsNMR={};
elengthsNMR={};
efractionsMR={};
elengthsMR={};
For[i=1,i<31,i++,
eselectionsNMR=RandomInteger[{1,Length[NMR]},100000];
eselectionsMR=RandomInteger[{1,Length[MR]},100000];(*RandomInteger[{1,Length[MR]},Round[N[(100000/Length[NMR])*Length[MR]]]];*)
etext={};
efractionNMR={};
elengthNMR={};
efractionMR={};
elengthMR={};
etext={};
etextwords={};
estringslist={};
For[a=1,a≤100000(*Length[selectionsNMR]*),a++, (*a is the entry number*)
emoji=0;
etext=NMR[[eselectionsNMR[[a]]]](*[[a]]*)[[2]];
(*CHECK FOR ENGLISH*)
If [LanguageIdentify[etext]==Entity["Language","English"],
etext=StringReplace[etext,"\\n"->" "]; (*Remove paragraph breaks*)
etextwords=TextWords[etext];
For [j=1,j<=Length[etextwords],j++, (*j is the particular word number*)
ewordstring=ToString[etextwords[[j]]];
If [SubsetQ[Characters[ewordstring],{":"}]&&Not[SubsetQ[Characters[ewordstring],{"."}]],emoji=emoji+1]];
If [Length[etextwords]≠0,AppendTo[efractionNMR,N[emoji/Length[etextwords]]]];
];
emoji=0;
etext=MR[[eselectionsMR[[a]]]](*[[a]]*)[[2]];
(*CHECK FOR ENGLISH*)
If [LanguageIdentify[etext]==Entity["Language","English"],
etext=StringReplace[etext,"\\n"->" "]; (*Remove paragraph breaks*)
etextwords=TextWords[etext];
For [j=1,j<=Length[etextwords],j++, (*j is the particular word number*)
ewordstring=ToString[etextwords[[j]]];
If [SubsetQ[Characters[ewordstring],{":"}]&&Not[SubsetQ[Characters[ewordstring],{"."}]],emoji=emoji+1]];
If [Length[etextwords]≠0,AppendTo[efractionMR,N[emoji/Length[etextwords]]]];
];
];
AppendTo[efractionsNMR,N[Mean[efractionNMR]]];
AppendTo[efractionsMR,N[Mean[efractionMR]]];
];

N[Dot[efractionsMR[[1;;31-1]],(*.*)(1/efractionsNMR[[1;;31-1]])]*1/(31-1)]

N[Mean[efractionMR]/Mean[efractionNMR]]

a

95172

efractionsNMR={0.004080773311299893`,0.003992478082879936`,0.003861283583579212`,0.003898699474962926`,0.003774381661451586`,0.0038623247290090162`,0.0037465439646440376`,0.0038960430275680386`,0.003938686193896197`,0.003967173343921169`,0.0041416208251161745`,0.003808707529124476`,0.003856731443417233`,0.003905351381186625`,0.0040646161417427365`,0.003996644140764362`,0.0039319232405872285`,0.0038510585880344094`,0.003926109799170121`,0.003970839771510237`,0.0038770474599120167`,0.003970208143964286`,0.0039949877417175175`,0.004011602052332143`,0.003929151105479037`,0.0041034587992714545`,0.003922289651233034`,0.0038659005205265626`,0.003914171250642558`,0.004010684314561049`}

efractionsMR={0.003989608480018545`,0.003985617732394662`,0.004282024950406424`,0.004010833463292132`,0.004229996017889794`,0.003943092040872571`,0.004029139853684299`,0.0040412835537068`,0.003931436012225808`,0.003957382060758107`,0.003757721481853125`,0.004014047086186865`,0.0040224418009298935`,0.004264301227961262`,0.003948739209726622`,0.003905758715967934`,0.0039424012536167004`,0.003957021309710878`,0.004039743954377808`,0.004133420815487368`,0.004082447661378138`,0.0038918073155880277`,0.004151577746737873`,0.003861519333860232`,0.003973684492029863`,0.0038016289776312715`,0.004069831874181648`,0.004088738249053665`,0.003835095545732315`,0.003995725438339339`}

(*Table[N[Dot[efractionsMR[[1;;m]],(*.*)(1/efractionsNMR[[1;;m]])]*1/(m)],{m,i}]*)

i

1/(Mean[efractionsNMR]/Mean[efractionsMR])

Variance[efractionsNMR]

Variance[efractionsMR]

Needs["HypothesisTesting`"]

MeanCI[efractionsNMR]

MeanCI[efractionsMR]

Histogram[{efractionsMR,efractionsNMR},ChartLegends->{"MR","NMR"}]

QuantilePlot[efractionsMR,efractionsNMR,PlotLegends->{"NMR","MR"}]

LocationEquivalenceTest[{efractionsNMR,efractionsMR},{"TestDataTable",All}]

"bs"=="bs"||"BS"

So there is an increase in average incidence when just looking as simple emojis in MR vs NMR, but the ANOVA p-value is not below 0.05 (and the confidence intervals overlap)