1function [X,Time,Z] = load_data(datafile,Spec,sample,loadExcel)
2%loadData Load vintage of data from file and format as structure
3%
4% Description:
5%
6%   Load data from file
7%
8% Input Arguments:
9%
10%   datafile - filename of Microsoft Excel workbook file
11%
12% Output Arguments:
13%
14% Data - structure with the following fields:
15%   .    X : T x N numeric array, transformed dataset
16%   . Time : T x 1 numeric array, date number with observation dates
17%   .    Z : T x N numeric array, raw (untransformed) dataset
18
19fprintf('Loading data... \n');
20
21if nargin < 4
22    loadExcel = 0;
23end
24
25ext = datafile(find(datafile=='.',1,'last'):end); % file extension
26
27idx = strfind(datafile,filesep); idx = idx(2);
28datafile_mat = fullfile(datafile(1:idx),'mat',[strtok(datafile(idx+1:end),'.') '.mat']);
29if exist(datafile_mat,'file')==2 && ~loadExcel
30    % Load raw data from MATLAB formatted binary (.mat) file
31    d = load(datafile_mat);
32    Z    = d.Z;
33    Time = d.Time;
34    Mnem = d.Mnem;
35elseif any(strcmpi(ext,{'.xlsx','.xls'}))
36    % Read raw data from Excel file
37    [Z,Time,Mnem] = readData(datafile);
38    save(datafile_mat,'Z','Time','Mnem');
39    %    Z : raw (untransformed) observed data
40    % Time : observation periods for the time series data
41    % Mnem : series ID for each variable
42else
43    error('Only Microsoft Excel workbook files supported.');
44end
45
46% Sort data based on model specification
47Z = sortData(Z,Mnem,Spec);
48clear Mnem; % since now Mnem == Spec.SeriesID
49
50% Transform data based on model specification
51[X,Time,Z] = transformData(Z,Time,Spec);
52
53% Drop data not in estimation sample
54if nargin == 3
55    [X,Time,Z] = dropData(X,Time,Z,sample);
56end
57
58end
59
60
61function [Z,Time,Mnem] = readData(datafile)
62%readData Read data from Microsoft Excel workbook file
63
64[DATA,TEXT] = xlsread(datafile,'data');
65Mnem = TEXT(1,2:end);
66if ispc
67    Time = datenum(TEXT(2:end,1),'mm/dd/yyyy');
68    Z    = DATA;
69else
70    Time = DATA(:,1) + datenum(1899,12,31) - 1;
71    Z    = DATA(:,2:end);
72end
73
74end
75
76function [Z,Mnem] = sortData(Z,Mnem,Spec)
77%sortData Sort series by order of model specification
78
79% Drop series not in Spec
80inSpec = ismember(Mnem,Spec.SeriesID);
81Mnem(~inSpec) = [];
82Z(:,~inSpec)  = [];
83
84% Sort series by ordering of Spec
85N = length(Spec.SeriesID);
86permutation = NaN(1,N);
87for i = 1:N
88    idxSeriesSpec = find(strcmp(Spec.SeriesID{i},Mnem));
89    permutation(i) = idxSeriesSpec;
90end
91
92Mnem = Mnem(permutation);
93Z    = Z(:,permutation);
94
95
96end
97
98
99function [X,Time,Z] = transformData(Z,Time,Spec)
100%transformData Transforms each data series based on Spec.Transformation
101%
102% Input Arguments:
103%
104%      Z : T x N numeric array, raw (untransformed) observed data
105%   Spec : structure          , model specification
106%
107% Output Arguments:
108%
109%      X : T x N numeric array, transformed data (stationary to enter DFM)
110
111T = size(Z,1);
112N = size(Z,2);
113
114X = NaN(T,N);
115for i = 1:N
116    formula = Spec.Transformation{i};
117    freq    = Spec.Frequency{i};
118    switch freq  % time step for different frequencies based on monthly time
119        case 'm' % Monthly
120            step = 1;
121        case 'q' % Quarterly
122            step = 3;
123    end
124    t1 = step;    % assume monthly observations start at beginning of quarter
125    n  = step/12; % number of years, needed to compute annual % changes
126    series  = Spec.SeriesName{i};
127    switch formula
128        case 'lin' % Levels (No Transformation)
129            X(:,i) = Z(:,i);
130        case 'chg' % Change (Difference)
131            X(t1:step:T,i) = [NaN; Z(t1+step:step:T,i) - Z(t1:step:T-t1,i)];
132        case 'ch1' % Year over Year Change (Difference)
133            X(12+t1:step:T,i) = Z(12+t1:step:T,i) - Z(t1:step:T-12,i);
134        case 'pch' % Percent Change
135            X(t1:step:T,i) = 100*[NaN; Z(t1+step:step:T,i) ./ Z(t1:step:T-t1,i) - 1];
136        case 'pc1' % Year over Year Percent Change
137            X(12+t1:step:T,i) = 100*(Z(12+t1:step:T,i) ./ Z(t1:step:T-12,i) - 1);
138        case 'pca' % Percent Change (Annual Rate)
139            X(t1:step:T,i) = 100*[NaN; (Z(t1+step:step:T,i) ./ Z(t1:step:T-step,i)).^(1/n) - 1];
140        case 'log' % Natural Log
141            X(:,i) = log(Z(:,i));
142        otherwise
143            warning(['Transformation ''' formula ''' not found for ' series '. ' 'Using untransformed data.']);
144            X(:,i) = Z(:,i);
145    end
146end
147
148
149% Drop first quarter of observations
150% since transformations cause missing values
151Time = Time(4:end);
152Z    = Z(4:end,:);
153X    = X(4:end,:);
154
155end
156
157
158function [X,Time,Z] = dropData(X,Time,Z,sample)
159%dropData Remove data not in estimation sample
160
161idxDrop = Time < sample;
162
163Time(idxDrop) = [];
164X(idxDrop,:)  = [];
165Z(idxDrop,:)  = [];
166
167end
168
169