From 0f734784c7d55da8f10c3d807a82a2e7fd3e3b00 Mon Sep 17 00:00:00 2001 From: flip phillips Date: Sun, 9 Sep 2018 21:33:40 -0400 Subject: [PATCH] Tweaked versioning, added stats, parallel image stuff --- .vscode/tasks.json | 6 ++++++ CHANGELOG.md | 7 +++++++ FPTools/Data.wl | 2 ++ FPTools/FPTools.wl | 2 +- FPTools/Image.wl | 19 +++++++++++++++++++ FPTools/Introspection.wl | 2 ++ FPTools/PacletInfo.m | 19 ++++++++++++++++--- FPTools/Statistics.wl | 34 ++++++++++++++++++++++++++++++++++ FPTools/Sugar.wl | 4 ++-- FPTools/Usage.wl | 25 ++++++++++++++++++++++++- FPTools/Versioning.wl | 4 +++- TODO.md | 4 ++++ build.wls | 2 +- release.sh | 10 +++++++++- version.json | 6 +++--- 15 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 FPTools/Statistics.wl diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 305c558..7d74fbe 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -17,6 +17,12 @@ "type": "shell", "command": "/usr/bin/env wolframscript -f install.wls", "group": "build" + }, + { + "label": "release", + "type": "shell", + "command": "./release.sh", + "group": "build" } ] } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index ea440a4..31239e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,14 @@ --> ## [Unreleased changes] +### Added +- `Statistics.wl` - routines from the pre v5 stats packages +- `Overlap` - where do two lists of events overlap. There's probably a time-series function that does this but I just have data and don't want to have to cram it into a TS. (e.g., my data is spatial so TS is the wrong metaphor) +- `ParallelImageApplyIndexed` +- `GetCompilerFunctions` +### Fixed +- missing trailing ; in `Sugar.wl` * * * diff --git a/FPTools/Data.wl b/FPTools/Data.wl index 5b637b2..e3f7599 100644 --- a/FPTools/Data.wl +++ b/FPTools/Data.wl @@ -30,3 +30,5 @@ ImportAsDataset[f_, True, Dataset[raw]]] + +Overlap[d1_,d2_]:= {Max[{Min[d1], Min[d2]}],Min[{Max[d1], Max[d2]}]} diff --git a/FPTools/FPTools.wl b/FPTools/FPTools.wl index d811ed2..fa418df 100644 --- a/FPTools/FPTools.wl +++ b/FPTools/FPTools.wl @@ -14,7 +14,7 @@ Module[{files}, (* load in any session *) files={"Introspection.wl","Entity.wl","Image.wl","Internet.wl","Sugar.wl", - "Versioning.wl","Globals.wl","Graphics.wl","Data.wl","Signals.wl","GA.wl"}; + "Versioning.wl","Globals.wl","Graphics.wl","Data.wl","Signals.wl","GA.wl","Statistics.wl"}; Map[Get[FileNameJoin[{DirectoryName[$InputFileName], #}]] &, files]; diff --git a/FPTools/Image.wl b/FPTools/Image.wl index 94635ce..fe9abce 100644 --- a/FPTools/Image.wl +++ b/FPTools/Image.wl @@ -164,3 +164,22 @@ LensDistortionCorrection[i_,{tx_,ty_},\[Theta]_,{p1_,p2_},kVec_,opt:OptionsPatte rt=ut@*RotationTransform[\[Theta]]; ImageTransformation[i,rt[LensDistortPoint[tt[#],{p1,p2},kVec]]&,PlotRange->{{0.1,0.9},{0.1,0.5}},opt]] + +(* this is still in development-ville, right now it is for a single image plane *) + +ParallelImageApplyIndexed[f_, img_] := + Module[{rows, cols, ix, id, decode, pd, res}, + {rows, cols} = ImageDimensions[img]; + + decode[i_] := {Quotient[i - 1, rows] + 1, Mod[i - 1, rows] + 1}; + + (*ix=Table[{r,c},{r,1,cols},{c,1,rows}];*) + + ix = Table[i, {i, 1, rows*cols}]; + id = Flatten[ImageData[img]]; + + pd = Transpose[{id, ix}]; + + res = ParallelMap[f[#[[1]], decode[#[[2]]]] &, pd]; + Image[Partition[res, rows]] + ] \ No newline at end of file diff --git a/FPTools/Introspection.wl b/FPTools/Introspection.wl index ef65b6c..0253967 100644 --- a/FPTools/Introspection.wl +++ b/FPTools/Introspection.wl @@ -124,3 +124,5 @@ GetSymbolsDefinedInMX[mxfile_] := Module[ ]; If[res=={{"System`Private`ConvertersPrivateDumpSymbol"}},None,res] ]; + +GetCompilerFunctions[] := Sort[Compile`CompilerFunctions[]] \ No newline at end of file diff --git a/FPTools/PacletInfo.m b/FPTools/PacletInfo.m index b673f68..6b5e974 100644 --- a/FPTools/PacletInfo.m +++ b/FPTools/PacletInfo.m @@ -5,8 +5,8 @@ Publisher -> "Skidmore Vision Lab", Copyright -> "©2018- Flip Phillips & Misc Contributors", License -> "MIT", - Version -> "1.0.4", - BuildNumber -> "160", + Version -> "1.0.6", + BuildNumber -> "185", MathematicaVersion -> "11.2+", URL -> "https://github.com/flipphillips/FPTools", Thumbnail -> "Documentation/icon.png", @@ -21,7 +21,7 @@ "FPTools`$FPToolsVersion","FPTools`$FPToolsDebug", (* data *) - "FPTools`ImportAsDataset", + "FPTools`ImportAsDataset", "FPTools`Overlap", (* Entity *) "FPTools`EntityPropertiesDataset", @@ -38,6 +38,7 @@ "FPTools`BlurFaces","FPTools`FindFaceImages", "FPTools`AddAlphaChannel", "FPTools`LensDistortPoint", "FPTools`LensDistortionCorrection", + "FPTools`ParallelImageApplyIndexed", (* ffmpeg *) "FPTools`$FFMpegPath", "FPTools`ImportMP4Frame", "FPTools`ImportMP4Frames", "FPTools`ImportWebFrame", @@ -48,6 +49,7 @@ (* introspection *) "FPTools`Spelunk","FPTools`PropertiesAvailability","FPTools`PropertiesDataset", "FPTools`NewKernelEvaluate","FPTools`GetSymbolsDefinedInMX", + "FPTools`GetCompilerFunctions", (* signals *) "FPTools`FindZeroCrossings","FPTools`UnitizedGaussianPDF", @@ -55,6 +57,17 @@ "FPTools`InverseFFTShift", "FPTools`RealSphericalHarmonicY", + (* statistics *) + "FPTools`LocationReport", + "FPTools`ShapeReport", + "FPTools`DispersionReport", + "FPTools`StandardDeviationMLE", + "FPTools`VarianceMLE", + "FPTools`StandardErrorOfSampleMean", + "FPTools`VarianceOfSampleMean", + "FPTools`CoefficientOfVariation", + "FPTools`ZeroMean", + (* sugar *) "FPTools`ApplyIf", "FPTools`ToCartesianRules", diff --git a/FPTools/Statistics.wl b/FPTools/Statistics.wl new file mode 100644 index 0000000..5495f6a --- /dev/null +++ b/FPTools/Statistics.wl @@ -0,0 +1,34 @@ + +(* some routines I used to use from the old statistics packages *) + +LocationReport[data_] := {Mean -> Mean[data], + HarmonicMean -> HarmonicMean[data], Median -> Median[data]} /; VectorQ[data] + +ShapeReport[list_] := { + Skewness -> Skewness[list] , + QuartileSkewness -> QuartileSkewness[list] , + Kurtosis -> Kurtosis[list]} /; VectorQ[list] + +DispersionReport[list_] := + { + Variance -> Variance[list] , + StandardDeviation -> StandardDeviation[list] , + MinMax -> MinMax[list] , + MeanDeviation -> MeanDeviation[list] , + MedianDeviation -> MedianDeviation[list] , + QuartileDeviation -> QuartileDeviation[list] + } /; VectorQ[list] + +StandardDeviationMLE[data_] := + StandardDeviation[data] Sqrt[(Length[data] - 1)/Length[data]] + +VarianceMLE[data_] := Variance[data] (Length[data] - 1)/Length[data] + +StandardErrorOfSampleMean[data_] := + StandardDeviation[data]/Sqrt[Length[data]] + +VarianceOfSampleMean[data_] := Variance[data]/Length[data] + +CoefficientOfVariation[data_] := StandardDeviation[data]/Mean[data] + +ZeroMean[data_] := data - Mean[data] diff --git a/FPTools/Sugar.wl b/FPTools/Sugar.wl index b86eedd..ccce1e1 100644 --- a/FPTools/Sugar.wl +++ b/FPTools/Sugar.wl @@ -7,9 +7,9 @@ ApplyIf[f_, g_][x_] := ApplyIf[f, g, x]; ToCartesianRules[] := MapThread[ - Rule, {{r, \[Theta], \[Phi]}, ToSphericalCoordinates[{x, y, z}]}] + Rule, {{r, \[Theta], \[Phi]}, ToSphericalCoordinates[{x, y, z}]}]; ToSphericalRules[] := MapThread[ Rule, {{x, y, z}, - FromSphericalCoordinates[{r, \[Theta], \[Phi]}]}]; \ No newline at end of file + FromSphericalCoordinates[{r, \[Theta], \[Phi]}]}]; diff --git a/FPTools/Usage.wl b/FPTools/Usage.wl index 3951e1b..d5fa456 100644 --- a/FPTools/Usage.wl +++ b/FPTools/Usage.wl @@ -20,6 +20,8 @@ PropertiesDataset::usage = "PropertiesDataset[object] returns a TableForm of the NewKernelEvaluate::usage = "NewKernelEvaluate[expr] evaluates expr in a fresh, clean kernel and returns the result."; GetSymbolsDefinedInMX::usage = "GetSymbolsDefinedInMX[mxfile] returns a list of symbols defined in mxfile."; +GetCompilerFunctions::usage = "GetCompilerFunctions[] returns the list of functions that are compilable."; + (* ::Section:: *) (*Entity*) @@ -48,6 +50,7 @@ LensDistortPoint::usage = "LensDistortPoint[{x,y},{p1,p2},{k1,...}] projects poi LensDistortionCorrection::usage = "LensDistortionCorrection[image,{tx,ty},theta,{p1,p1},{k1,k2,k3...}] performs a lens distortion correction along with a generalized translation (t) and rotation (theta). p1 and p2 specify 1st order astygmatism and k1-6 the different orders of distortion beyond that. Note you only need to supply k's up to the desired order of correction."; +ParallelImageApplyIndexed::usage = "ParallelImageApplyIndexed[f,im] maps f over im using parallel kernels. At this point, its only a single image plane"; (* ::Section:: *) (*Internet*) @@ -87,6 +90,8 @@ ImportAsDataset::usage = "ImportAsDataset[f] imports f and uses the first row as UnitizedGaussianPDF::usage = "UnitizedGaussian[mu,sigma] gives a pure function of a PDF whose value at the mean = 1.0."; +Overlap::usage = "Overlap[l1,l2] gives the intersection of the range of two lists."; + (* ::Section:: *) (*Signals*) @@ -102,4 +107,22 @@ RealSphericalHarmonicY::usage = "RealSphericalHarmonicY[l,m,th,phi] returns real GARepopulate::usage = "GARepopulate[pop,fitfunc] creates a new population using fitfunc."; GAMutate::usage = "GAMutate[chromo] mutates a chromosome."; -GACrossover::usage = "GACrossover[c1,c2] performs a random crossover of c1 & c2. Returns two offspring."; \ No newline at end of file +GACrossover::usage = "GACrossover[c1,c2] performs a random crossover of c1 & c2. Returns two offspring."; + +(* ::Section:: *) +(*Statistics*) +LocationReport::usage = "LocationReport[list] gives the Mean, HarmonicMean, and Median location statistics for list."; +DispersionReport::usage = "DispersionReport[list] gives the Variance, StandardDeviation, SampleRange, MeanDeviation, MedianDeviation, and QuartileDeviation dispersion statistics for list."; +ShapeReport::usage = "ShapeReport[list] gives the Skewness, QuartileSkewness, and KurtosisExcess shape statistics for list."; + +StandardDeviationMLE::usage = "StandardDeviationMLE[list] gives the standard deviation of the entries in list. Division by n is used, giving a maximum likelihood estimate of the population standard deviation."; + +VarianceMLE::usage = "VarianceMLE[list] gives the variance of the entries in list. Division by n (rather than n-1) is used, giving a maximum likelihood estimate of the population variance (use Variance[list] for an unbiased estimate)."; + +StandardErrorOfSampleMean::usage = "StandardErrorOfSampleMean[list] gives an unbiased estimate of the standard error (standard deviation) of the sample mean, using the entries in list as a sample from the population."; + +VarianceOfSampleMean::usage = "VarianceOfSampleMean[list] gives an unbiased estimate of the variance of the sample mean, using the entries in list as a sample from the population."; + +CoefficientOfVariation::usage = "CoefficientOfVariation[list] gives the coefficient of variation, defined as the ratio of the standard deviation to the mean of the entries in list. If the entries are not all positive, then $Failed is returned." + +ZeroMean::usage = "Zeroize the data about the mean."; diff --git a/FPTools/Versioning.wl b/FPTools/Versioning.wl index 3dff1c9..3cfb5ab 100644 --- a/FPTools/Versioning.wl +++ b/FPTools/Versioning.wl @@ -8,7 +8,7 @@ VersionInformation[path_] := Module[{vp}, Association[Import[vp]], None]] -VersionString[path_, OptionsPattern[{"Build" -> False}]] := +VersionString[path_, OptionsPattern[{"Build"->False,"Prerelease"->False}]] := Module[{vi, vs}, vi = VersionInformation[path]; If[vi === None, Return[""]]; @@ -18,6 +18,8 @@ VersionString[path_, OptionsPattern[{"Build" -> False}]] := vi /@ {"major_version", "minor_version", "revision_number"}, "."]]; + vs = If[OptionValue["Prerelease"]&&vi["prerelease"],vs<>"pre",vs]; + If[OptionValue["Build"], vs <> " build " <> ToString[vi["build_number"]], vs]] diff --git a/TODO.md b/TODO.md index e67a936..3b890fa 100644 --- a/TODO.md +++ b/TODO.md @@ -34,3 +34,7 @@ ### release.sh - make use pre-release setting for sending to GitHub + +### Improve ParallelImageApplyIndexed + +### Make 'UpdateFlipTools' diff --git a/build.wls b/build.wls index 2d859bc..0def1d3 100755 --- a/build.wls +++ b/build.wls @@ -16,7 +16,7 @@ VersionBumpBuild[$directory]; (* globals *) Unprotect[{$FPToolsDebug,$FPToolsVersion}]; $FPToolsDebug=False; -$FPToolsVersion=VersionString[$directory,"Build"->True]; +$FPToolsVersion=VersionString[$directory,"Build"->True,"Prerelease"->True]; (* create global file *) gfile=FileNameJoin[{$directory,$packageName,$globalsFile}]; diff --git a/release.sh b/release.sh index 4bba31f..d92024a 100755 --- a/release.sh +++ b/release.sh @@ -6,12 +6,20 @@ source ~/.bash_aliases if [[ -e ./${VERSIONFILE} ]] ; then PACLET=`jq -r .name ${VERSIONFILE}` V=`versionstring` + VF=`fullversionstring` + PRERELEASE=`jq -r .prerelease ${VERSIONFILE}` PACLETFILE="${PACLET}-${V}.paclet" + if [[ ${PRERELEASE} == "true" ]] ; then + FLAGS="-p" + else + FLAGS="" + fi + echo "Releasing $PACLET version $V as $PACLETFILE" if [[ -e ${PACLETFILE} ]] ; then - hub release create -a ${PACLETFILE} v${V} + hub release create ${FLAGS} -a ${PACLETFILE} v${VF} else echo "release: No paclet file ${PACLETFILE}." exit 2 diff --git a/version.json b/version.json index d422fef..229ad01 100644 --- a/version.json +++ b/version.json @@ -3,7 +3,7 @@ "prerelease": true, "major_version": 1, "minor_version": 0, - "revision_number": 4, - "build_number": 161, - "date": "Sat Aug 4 17:15:45 EDT 2018" + "revision_number": 7, + "build_number": 186, + "date": "Sun Sep 9 21:33:40 EDT 2018" }