1. Creating histogram-valued data

library(HistDAWass)

In HistDAWass a histogram is a distributionH object and a data table whose cells are histograms is a MatH object: rows are individuals (or groups) and columns are histogram-valued variables. This vignette shows how to build them.

A single histogram

A histogram is described by its cumulative distribution function (cdf): the bounds of the bins x and the values p of the cdf at the bounds. For example, the histogram

Bin	Probability	cdf
[1, 2)	0.4	0.4
[2, 3]	0.6	1.0

is created as

d1 <- distributionH(x = c(1, 2, 3), p = c(0, 0.4, 1))
d1
#>           X         p
#> Bin_1 [ 1 ; 2 )       0.4
#> Bin_2 [ 2 ; 3 ]       0.6
#> 
#>  mean =  2.1   std  =  0.568624070307733 
#>

x must be non-decreasing and p must be non-decreasing from 0 to 1. The mean and the standard deviation are computed when the object is created and are stored in the slots m and s:

d1@m
#> [1] 2.1
d1@s
#> [1] 0.5686241

A single value defines an impulse (a degenerate distribution), and a bin with zero probability (an empty bin) gets a tiny weight of 1e-8, to avoid numerical problems:

distributionH(5)@x
#> [1] 5 5
d2 <- distributionH(x = c(0, 1, 2, 4), p = c(0, 0.5, 0.5, 1))
diff(d2@p) # the probabilities of the bins
#> [1] 5e-01 1e-08 5e-01

plot(d1)

From raw data to a histogram

data2hist() builds a histogram from a sample of values. Several algorithms are available for choosing the bins:

set.seed(1)
x <- c(rnorm(300, mean = 10, sd = 2), rnorm(200, mean = 18, sd = 1))

h_base <- data2hist(x, algo = "base")                     # the bins of hist()
h_quan <- data2hist(x, algo = "FixedQuantiles", qua = 10) # 10 equal-probability bins
h_man  <- data2hist(x, algo = "ManualBreaks", breaks = seq(0, 25, by = 2.5))
h_pl   <- data2hist(x, algo = "PolyLine", epsilon = 0.01) # piecewise linear cdf
sapply(list(base = h_base, quantiles = h_quan, manual = h_man, polyline = h_pl),
       function(h) length(h@x) - 1) # number of bins
#>      base quantiles    manual  polyline 
#>         9        10         8        15

The default, algo = "histogram", chooses the bins with the penalised likelihood method of the histogram package: it is accurate but slower for large samples.

plot(h_quan)

plot(h_quan, type = "CDF")

A matrix of histograms

A MatH object is created from a list of distributionH objects, filled by columns (default) or by rows:

ListOfDist <- list(
  distributionH(c(1, 2, 3), c(0, 0.4, 1)),
  distributionH(c(7, 8, 10, 15), c(0, 0.2, 0.7, 1)),
  distributionH(c(9, 11, 20), c(0, 0.5, 1)),
  distributionH(c(2, 5, 8), c(0, 0.3, 1)),
  distributionH(c(8, 10, 15), c(0, 0.75, 1)),
  distributionH(c(20, 22, 24), c(0, 0.12, 1))
)
MyMAT <- MatH(x = ListOfDist, nrows = 3, ncols = 2,
              rownames = c("I1", "I2", "I3"), varnames = c("Var1", "Var2"))
MyMAT
#> a matrix of distributions 
#>  2  variables  3  rows 
#>  each distibution in the cell is represented by the mean and the standard deviation 
#>               Var1                     Var2          
#> I1  [m= 2.1  ,s= 0.56862 ]   [m= 5.6  ,s= 1.6248 ]  
#> I2  [m= 9.75  ,s= 2.0907 ]  [m= 9.875  ,s= 1.7515 ] 
#> I3 [m= 12.75  ,s= 3.3323 ]  [m= 22.76  ,s= 0.86933 ]

From a data frame of raw data

The most common case: a data frame of raw observations and a grouping variable. Each group becomes a row of the MatH, each numeric variable a column. With the iris data, the three species are the individuals:

groups <- levels(iris$Species)
vars <- names(iris)[1:4]
cells <- list()
for (v in vars) {           # the matrix is filled by columns
  for (g in groups) {
    cells <- c(cells, data2hist(iris[iris$Species == g, v],
                                algo = "FixedQuantiles", qua = 5))
  }
}
IRIS <- MatH(x = cells, nrows = length(groups), ncols = length(vars),
             rownames = groups, varnames = vars)
IRIS
#> a matrix of distributions 
#>  4  variables  3  rows 
#>  each distibution in the cell is represented by the mean and the standard deviation 
#>               Sepal.Length             Sepal.Width              Petal.Length             Petal.Width       
#> setosa [m= 5.026  ,s= 0.36823 ] [m= 3.414  ,s= 0.46861 ] [m= 1.45  ,s= 0.20857 ]  [m= 0.25  ,s= 0.11328 ] 
#> versicolor [m= 5.918  ,s= 0.53018 ] [m= 2.76  ,s= 0.33427 ]  [m= 4.27  ,s= 0.51617 ]  [m= 1.356  ,s= 0.19626 ]
#> virginica [m= 6.56  ,s= 0.71396 ]  [m= 2.96  ,s= 0.36203 ]  [m= 5.552  ,s= 0.57681 ] [m= 2.01  ,s= 0.28501 ]

plot(IRIS)                  # histograms

plot(IRIS, type = "DENS")   # smoothed densities
#> Picking joint bandwidth of 0.0933
#> Picking joint bandwidth of 0.0405
#> Picking joint bandwidth of 0.118
#> Picking joint bandwidth of 0.0739

All the plot methods return a ggplot object (invisibly), that can be modified with the functions of ggplot2:

p <- plot(IRIS, type = "BOXPLOT")

p + ggplot2::ggtitle("Iris data: boxplots of the histograms")

Working with a MatH

Rows and columns are extracted as for an ordinary matrix, and matrices with the same columns (rows) can be bound by rows (columns):

dim(BLOOD@M)
#> [1] 14  3
B <- BLOOD[1:4, c("Cholesterol", "Hemoglobin")]
B
#> a matrix of distributions 
#>  2  variables  4  rows 
#>  each distibution in the cell is represented by the mean and the standard deviation 
#>                  Cholesterol               Hemoglobin        
#> u1: F-20  [m= 150.1  ,s= 26.336 ]  [m= 13.695  ,s= 0.55031 ]
#> u2: F-30 [m= 150.71  ,s= 25.284 ]  [m= 12.158  ,s= 0.52834 ]
#> u3: F-40 [m= 164.96  ,s= 25.334 ]  [m= 12.134  ,s= 0.50739 ]
#> u4: F-50 [m= 186.51  ,s= 26.655 ]  [m= 12.133  ,s= 0.58514 ]
get.MatH.main.info(B)
#> $nrows
#> [1] 4
#> 
#> $ncols
#> [1] 2
#> 
#> $rownames
#> [1] "u1: F-20" "u2: F-30" "u3: F-40" "u4: F-50"
#> 
#> $varnames
#> [1] "Cholesterol" "Hemoglobin"

WH.bind.row(BLOOD[1:2, ], BLOOD[13:14, ])
#> a matrix of distributions 
#>  3  variables  4  rows 
#>  each distibution in the cell is represented by the mean and the standard deviation 
#>                  Cholesterol               Hemoglobin                Hematocrit        
#> u1: F-20  [m= 150.1  ,s= 26.336 ]  [m= 13.695  ,s= 0.55031 ] [m= 41.526  ,s= 2.1968 ] 
#> u2: F-30 [m= 150.71  ,s= 25.284 ]  [m= 12.158  ,s= 0.52834 ] [m= 36.497  ,s= 2.1225 ] 
#> u13: M-70 [m= 205.66  ,s= 22.499 ]  [m= 12.15  ,s= 0.33425 ]   [m= 35.75  ,s= 2.1651 ] 
#> u14: M-80+ [m= 205.48  ,s= 23.537 ]   [m= 12.12  ,s= 0.6163 ]   [m= 38.45  ,s= 2.6158 ]

Single cells are read and written with get.cell.MatH() and set.cell.MatH() (or directly through the slot M):

get.cell.MatH(BLOOD, r = 1, c = 1)
#>               X             p
#> Bin_1  [ 80 ; 100 )         0.025
#> Bin_2 [ 100 ; 120 )         0.075
#> Bin_3 [ 120 ; 135 )         0.175
#> Bin_4 [ 135 ; 150 )          0.25
#> Bin_5 [ 150 ; 165 )           0.2
#> Bin_6 [ 165 ; 180 )         0.162
#> Bin_7 [ 180 ; 200 )         0.088
#> Bin_8 [ 200 ; 240 ]         0.025
#> 
#>  mean =  150.1025   std  =  26.3356126518827 
#> 
B2 <- set.cell.MatH(distributionH(c(100, 150, 200), c(0, 0.5, 1)), B, r = 1, c = 1)
B2@M[[1, 1]]
#>               X             p
#> Bin_1 [ 100 ; 150 )           0.5
#> Bin_2 [ 150 ; 200 ]           0.5
#> 
#>  mean =  150   std  =  28.8675134594813 
#>

Registration

Two histograms with different bins can be expressed on the same cdf values without changing them (registration). This is the basis of all the fast computations of the package and is done automatically by the methods; it can also be done explicitly:

R <- registerMH(MyMAT)
is.registeredMH(MyMAT)
#> [1] FALSE
is.registeredMH(R)
#> [1] TRUE
R@M[[1, 1]]@p
#> [1] 0.00 0.12 0.20 0.30 0.40 0.50 0.70 0.75 1.00

Histogram time series

A histogram time series (HTS) is a sequence of histograms with a time reference; RetHTS is an example:

RetHTS
#>     Tstamp T_period_start T_period_end          mean        std         skew
#> 1    38749          38749        38749 -0.0019860297 0.04224402  0.696353637
#> 2    38750          38750        38750 -0.0027525921 0.03422065  0.161710205
#> 3    38751          38751        38751 -0.0022545180 0.04252923  1.054658950
#> 4    38754          38754        38754 -0.0056181618 0.03660866 -0.287735490
#> 5    38755          38755        38755 -0.0108187759 0.04469958 -0.691931506
#> 6    38756          38756        38756 -0.0015679186 0.03932478  0.676302020
#> 7    38757          38757        38757 -0.0038849961 0.03521140  0.474190871
#> 8    38758          38758        38758 -0.0057090553 0.05180512 -0.108275213
#> 9    38761          38761        38761 -0.0048545204 0.03741751 -0.264327106
#> 10   38762          38762        38762 -0.0060801071 0.04140493  0.276796508
#> 11   38763          38763        38763 -0.0018814832 0.05313136 -0.151476128
#> 12   38764          38764        38764 -0.0060526111 0.03971651 -0.041866347
#> 13   38765          38765        38765 -0.0019454181 0.05084949  0.354931260
#> 14   38768          38768        38768 -0.0051052438 0.02793509  0.009895520
#> 15   38769          38769        38769 -0.0034561197 0.02806190  0.028340867
#> 16   38770          38770        38770 -0.0069337719 0.03726113 -0.602640682
#> 17   38771          38771        38771 -0.0106445600 0.05501497 -0.995706447
#> 18   38772          38772        38772 -0.0073090483 0.05280985 -0.254510473
#> 19   38775          38775        38775 -0.0092402357 0.04994473 -2.456202932
#> 20   38776          38776        38776 -0.0070208812 0.03425427 -0.170867642
#> 21   38777          38777        38777 -0.0047212118 0.04867714  0.296046999
#> 22   38778          38778        38778 -0.0047038051 0.03674708  0.240445851
#> 23   38779          38779        38779 -0.0064233064 0.05259779 -1.269210426
#> 24   38782          38782        38782 -0.0011817858 0.04103728  0.718818282
#> 25   38783          38783        38783 -0.0047734896 0.03857179 -0.141197527
#> 26   38784          38784        38784 -0.0048606093 0.03369879 -0.515454518
#> 27   38785          38785        38785 -0.0047037812 0.05545347 -0.295411527
#> 28   38786          38786        38786 -0.0007272310 0.05204268  0.707027815
#> 29   38789          38789        38789 -0.0064363434 0.03191699 -0.078502518
#> 30   38790          38790        38790 -0.0105923052 0.04003190 -0.199008788
#> 31   38791          38791        38791 -0.0069511893 0.03898982 -0.086497721
#> 32   38792          38792        38792 -0.0074389851 0.04508932 -0.935993029
#> 33   38793          38793        38793 -0.0075729608 0.04550285  0.000219057
#> 34   38796          38796        38796 -0.0030633416 0.04627440  0.093724269
#> 35   38797          38797        38797 -0.0017307386 0.04216918  0.617796078
#> 36   38798          38798        38798 -0.0052264546 0.03984408  0.118317228
#> 37   38799          38799        38799 -0.0010626723 0.04243500  2.076246948
#> 38   38800          38800        38800 -0.0087817694 0.05320716 -2.306995986
#> 39   38803          38803        38803 -0.0072534963 0.04214570 -1.085068170
#> 40   38804          38804        38804 -0.0002786968 0.05085144  1.087733108
#> 41   38805          38805        38805 -0.0051219166 0.04418986 -0.011704014
#> 42   38806          38806        38806 -0.0076828832 0.05040461 -1.028022895
#> 43   38807          38807        38807 -0.0060545003 0.05235379 -0.606038718
#> 44   38810          38810        38810 -0.0061090452 0.05826492 -0.395657824
#> 45   38811          38811        38811 -0.0055985506 0.04355256 -0.486466779
#> 46   38812          38812        38812 -0.0046689580 0.04185633  0.459999525
#> 47   38813          38813        38813 -0.0029616471 0.03672018 -0.042695220
#> 48   38814          38814        38814 -0.0041635993 0.04382331  0.448764601
#> 49   38817          38817        38817 -0.0043612930 0.03274848  0.187488595
#> 50   38818          38818        38818 -0.0057142672 0.03244375 -0.246011044
#> 51   38819          38819        38819 -0.0036585120 0.03779608  0.122059841
#> 52   38820          38820        38820 -0.0049477194 0.02559904  0.261164825
#> 53   38821          38821        38821 -0.0045724760 0.01996054  0.563223610
#> 54   38824          38824        38824 -0.0091635871 0.04472303  0.237533944
#> 55   38825          38825        38825 -0.0081532739 0.04128826 -0.939945016
#> 56   38826          38826        38826 -0.0046166875 0.04686141  0.537764380
#> 57   38827          38827        38827 -0.0029616389 0.03641048 -0.028545536
#> 58   38828          38828        38828 -0.0089010695 0.04042375  0.713792677
#> 59   38831          38831        38831 -0.0101929316 0.05653616 -1.690331963
#> 60   38832          38832        38832 -0.0028571058 0.04468332  0.083454199
#> 61   38833          38833        38833 -0.0060278319 0.04528931 -0.440157943
#> 62   38834          38834        38834 -0.0093705645 0.06338602 -3.139906218
#> 63   38835          38835        38835 -0.0048908788 0.03923458 -0.850739442
#> 64   38838          38838        38838 -0.0075879791 0.05844668 -0.987439643
#> 65   38839          38839        38839 -0.0040766233 0.04768498 -0.616224072
#> 66   38840          38840        38840 -0.0047901832 0.03987908 -0.187875569
#> 67   38841          38841        38841 -0.0050174516 0.04519448 -0.363053027
#> 68   38842          38842        38842 -0.0100362910 0.06474940 -3.844042297
#> 69   38845          38845        38845 -0.0069999632 0.04992085 -0.780782148
#> 70   38846          38846        38846 -0.0075086792 0.04351507 -0.119390300
#> 71   38847          38847        38847 -0.0098083081 0.05765520 -0.452096610
#> 72   38848          38848        38848 -0.0054180625 0.06859318 -1.126436266
#> 73   38849          38849        38849 -0.0075272232 0.06127886  0.827625388
#> 74   38852          38852        38852 -0.0041985656 0.06142446  0.529495001
#> 75   38853          38853        38853 -0.0087804475 0.05342095 -0.070909200
#> 76   38854          38854        38854 -0.0054529214 0.06511938  0.008851600
#> 77   38855          38855        38855 -0.0081010001 0.05995930 -0.677938766
#> 78   38856          38856        38856  0.0004364151 0.06453323  0.156490593
#> 79   38859          38859        38859 -0.0039895079 0.05471918 -0.333312689
#> 80   38860          38860        38860 -0.0028146368 0.05766031  0.328591403
#> 81   38861          38861        38861 -0.0048257411 0.06988168  0.611671245
#> 82   38862          38862        38862 -0.0083797562 0.04348548 -0.346286093
#> 83   38863          38863        38863 -0.0021897476 0.04553626  0.064172498
#> 84   38866          38866        38866 -0.0051851697 0.02586819 -0.160231468
#> 85   38867          38867        38867 -0.0056097161 0.05844088  0.599221592
#> 86   38868          38868        38868 -0.0041637175 0.05226989 -1.287457703
#> 87   38869          38869        38869 -0.0031184325 0.04704296 -0.807542954
#> 88   38870          38870        38870 -0.0099453839 0.06255275 -4.200081947
#> 89   38873          38873        38873 -0.0027655402 0.03808986  1.089654951
#> 90   38874          38874        38874 -0.0005419267 0.04214455  0.897775010
#> 91   38875          38875        38875 -0.0033971822 0.03736942 -0.093831117
#> 92   38876          38876        38876 -0.0036933386 0.04352173  0.834360678
#> 93   38877          38877        38877 -0.0062726803 0.04113597  0.501480836
#> 94   38880          38880        38880 -0.0043090723 0.03271514 -0.149798527
#> 95   38881          38881        38881 -0.0002787105 0.04565224  1.053359064
#> 96   38882          38882        38882 -0.0059789644 0.04801573 -1.297302438
#> 97   38883          38883        38883 -0.0067595325 0.04059745 -0.074733516
#> 98   38884          38884        38884 -0.0035948660 0.03331588 -0.289386820
#> 99   38887          38887        38887 -0.0036701835 0.03245845  0.535800150
#> 100  38888          38888        38888 -0.0074911337 0.03855696 -0.354157836
#> 101  38889          38889        38889 -0.0052787230 0.03079112  0.191272381
#> 102  38890          38890        38890 -0.0007491067 0.03511747  0.840005560
#> 103  38891          38891        38891 -0.0049817761 0.04259499 -0.213627962
#> 104  38894          38894        38894 -0.0061031761 0.03335348  0.516790674
#> 105  38895          38895        38895 -0.0040592063 0.03850145  0.089547785
#> 106  38896          38896        38896 -0.0051567757 0.02822681 -0.303361576
#> 107  38897          38897        38897 -0.0105069555 0.04670969 -2.181569432
#> 108  38898          38898        38898 -0.0077189437 0.04557718 -0.555615542
#>          kurt
#> 1    6.433505
#> 2    3.860462
#> 3    6.334018
#> 4    4.077056
#> 5    5.949878
#> 6    4.805117
#> 7    4.194543
#> 8    4.937425
#> 9    4.290259
#> 10   6.148626
#> 11   9.303889
#> 12   4.189392
#> 13   6.160713
#> 14   5.072376
#> 15   4.330262
#> 16   5.229702
#> 17   6.528730
#> 18   7.056863
#> 19  16.218037
#> 20   3.533228
#> 21   5.833572
#> 22   6.362662
#> 23  11.922934
#> 24   5.971927
#> 25   5.804973
#> 26   3.729712
#> 27   8.626437
#> 28   8.276109
#> 29   3.762579
#> 30   3.885703
#> 31   5.293574
#> 32   7.833595
#> 33   4.890152
#> 34   5.230596
#> 35   5.686636
#> 36   5.678807
#> 37  15.385329
#> 38  13.792963
#> 39   4.974648
#> 40   8.830547
#> 41   6.374511
#> 42   9.496948
#> 43   8.880585
#> 44   6.925153
#> 45   6.884867
#> 46   5.036577
#> 47   4.208889
#> 48   7.897283
#> 49   4.668999
#> 50   3.562023
#> 51   5.802111
#> 52   4.069244
#> 53   5.127033
#> 54   9.820100
#> 55   7.934035
#> 56   5.537140
#> 57   6.009552
#> 58   6.595900
#> 59  11.243804
#> 60   5.849040
#> 61   6.474160
#> 62  21.884934
#> 63   5.869432
#> 64   9.724247
#> 65   4.198066
#> 66   4.626865
#> 67   3.945153
#> 68  30.346126
#> 69   5.403221
#> 70   4.367429
#> 71   7.850489
#> 72   8.279600
#> 73   6.821111
#> 74   4.612564
#> 75   6.192405
#> 76   5.422810
#> 77   6.368277
#> 78   6.391666
#> 79   4.294341
#> 80   5.971598
#> 81   4.677875
#> 82   5.958471
#> 83   5.382289
#> 84   4.313987
#> 85   5.366215
#> 86   9.113494
#> 87   6.421688
#> 88  33.147580
#> 89   7.042168
#> 90   6.642544
#> 91   5.687776
#> 92   8.806083
#> 93   7.990728
#> 94   3.469891
#> 95   7.260356
#> 96  14.176536
#> 97   9.526191
#> 98   4.557180
#> 99   6.465983
#> 100  6.562610
#> 101  4.722468
#> 102  5.816567
#> 103  7.646801
#> 104  5.481879
#> 105  4.773725
#> 106  4.315656
#> 107 12.221400
#> 108  6.150324
plot(subsetHTS(RetHTS, from = 1, to = 15), type = "BOXPLOT")

See the other vignettes for the statistics (vignette("HistDAWass-02-basic-statistics")), clustering, principal component analysis and regression of histogram data.