Code 
projectdir = splitpath(pwd()) |>
    ss -> joinpath(ss[1:findall([s == "endogenous-macrodynamics-in-algorithmic-recourse" for s in ss])[1]]...) 
cd(projectdir)
Code 
models = [
    :LogisticRegression, 
    :FluxModel, 
    :FluxEnsemble
]
opt = Flux.Descent(0.01) 
generators = Dict(
    :Greedy=>GreedyGenerator(), 
    :Generic=>GenericGenerator(opt = opt),
    :REVISE=>REVISEGenerator(opt = opt),
    :DICE=>DiCEGenerator(opt = opt),
)
Code 
max_obs = 5000
data_sets = load_real_world(max_obs)
choices = [
    :cal_housing, 
    :credit_default, 
    :gmsc, 
]
data_sets = filter(p -> p[1] in choices, data_sets)
Code 
using CounterfactualExplanations.DataPreprocessing: unpack
bs = 500
function data_loader(data::CounterfactualData)
    X, y = unpack(data)
    data = Flux.DataLoader((X,y),batchsize=bs)
    return data
end
model_params = (batch_norm=false,n_hidden=64,n_layers=3,dropout=true,p_dropout=0.1)
Code 
experiments = set_up_experiments(
    data_sets,models,generators; 
    pre_train_models=100, model_params=model_params, 
    data_loader=data_loader
)

4.1 Running the Experiment

Code 
n_evals = 5
n_rounds = 50
evaluate_every = Int(round(n_rounds/n_evals))
n_folds = 5
n_samples = 10000
T = 100
generative_model_params = (epochs=250, latent_dim=8)
results = run_experiments(
    experiments;
    save_path=output_path,evaluate_every=evaluate_every,n_rounds=n_rounds, n_folds=n_folds, T=T, n_samples=n_samples,
    generative_model_params=generative_model_params
)
Serialization.serialize(joinpath(output_path,"results.jls"),results)

4.2 Results

Code 
results = load_real_world_results()

4.2.1 Animated GIFs

Code 
using Images
line_charts = Dict()
errorbar_charts = Dict()
for (data_name, res) in results
    plt = plot_res(res)
    Images.save(joinpath(www_path, "line_chart_$(data_name).png"), plt)
    Images.save(joinpath(www_paper_path, "line_chart_$(data_name).png"), plt)
    line_charts[data_name] = plt
    plt = plot_res(res,maximum(res.output.n))
    Images.save(joinpath(www_path, "errorbar_chart_$(data_name).png"), plt)
    Images.save(joinpath(www_paper_path, "errorbar_chart_$(data_name).png"), plt)
    errorbar_charts[data_name] = plt
end

4.2.2 Line Charts

The evolution of the evaluation metrics over the course of the experiment is shown for different datasets in Figure 4.1 to Figure 4.3.

Figure 4.1: Evolution of evaluation metrics over the course of the experiment. Data: California Housing.
Figure 4.2: Evolution of evaluation metrics over the course of the experiment. Data: Credit Default.
Figure 4.3: Evolution of evaluation metrics over the course of the experiment. Data: GMSC.

4.2.3 Error Bar Charts

The evaluation metrics at the end of the experiment are shown for different datasets in Figure 4.4 to Figure 4.6.

Figure 4.4: Evaluation metrics at the end of the experiment. Data: California Housing.
Figure 4.5: Evaluation metrics at the end of the experiment. Data: Credit Default.
Figure 4.6: Evaluation metrics at the end of the experiment. Data: GMSC.

4.2.4 Bootstrap

Code 
n_bootstrap = 100
df = run_bootstrap(results, n_bootstrap; filename=joinpath(output_path,"bootstrap.csv"))

Table 4.1 presents the tests for statistical significance of the estimated MMD metrics.

Code 
df = EMAR.aggregate_bs_real_world()
CSV.write(joinpath(www_path, "bootstrap.csv"), df)
CSV.write(joinpath(www_paper_path, "bootstrap.csv"), df)
EMAR.tabulate_bs(df, Val(:html))
Table 4.1: Tests for statistical significance of the estimated MMD metrics. We have highlighted p-values smaller than the significance level \(\alpha=0.05\) in bold. Data: Real-World.
Metric Data Generator Model p-value
MMD Cal Housing DICE Deep Ensemble 0.0
MMD Cal Housing DICE Linear 0.0
MMD Cal Housing DICE MLP 0.0
MMD Cal Housing Generic (γ=0.5) Deep Ensemble 0.0
MMD Cal Housing Generic (γ=0.5) Linear 0.0
MMD Cal Housing Generic (γ=0.5) MLP 0.0
MMD Cal Housing Greedy Deep Ensemble 0.0
MMD Cal Housing Greedy Linear 0.0
MMD Cal Housing Greedy MLP 0.0
MMD Cal Housing Latent Deep Ensemble 0.0
MMD Cal Housing Latent Linear 0.0
MMD Cal Housing Latent MLP 0.0
MMD Credit Default DICE Deep Ensemble 1.0
MMD Credit Default DICE Linear 1.0
MMD Credit Default DICE MLP 1.0
MMD Credit Default Generic (γ=0.5) Deep Ensemble 1.0
MMD Credit Default Generic (γ=0.5) Linear 1.0
MMD Credit Default Generic (γ=0.5) MLP 1.0
MMD Credit Default Greedy Deep Ensemble 1.0
MMD Credit Default Greedy Linear 1.0
MMD Credit Default Greedy MLP 1.0
MMD Credit Default Latent Deep Ensemble 0.0
MMD Credit Default Latent Linear 1.0
MMD Credit Default Latent MLP 0.0
MMD GMSC DICE Deep Ensemble 0.082
MMD GMSC DICE Linear 0.51
MMD GMSC DICE MLP 0.338
MMD GMSC Generic (γ=0.5) Deep Ensemble 0.306
MMD GMSC Generic (γ=0.5) Linear 0.278
MMD GMSC Generic (γ=0.5) MLP 0.128
MMD GMSC Greedy Deep Ensemble 0.032
MMD GMSC Greedy Linear 0.006
MMD GMSC Greedy MLP 0.0
MMD GMSC Latent Deep Ensemble 0.0
MMD GMSC Latent Linear 0.0
MMD GMSC Latent MLP 0.0
PP MMD Cal Housing DICE Deep Ensemble 0.0
PP MMD Cal Housing DICE Linear 0.0
PP MMD Cal Housing DICE MLP 0.0
PP MMD Cal Housing Generic (γ=0.5) Deep Ensemble 0.0
PP MMD Cal Housing Generic (γ=0.5) Linear 0.0
PP MMD Cal Housing Generic (γ=0.5) MLP 0.0
PP MMD Cal Housing Greedy Deep Ensemble 0.0
PP MMD Cal Housing Greedy Linear 0.0
PP MMD Cal Housing Greedy MLP 0.0
PP MMD Cal Housing Latent Deep Ensemble 0.0
PP MMD Cal Housing Latent Linear 0.0
PP MMD Cal Housing Latent MLP 0.0
PP MMD Credit Default DICE Deep Ensemble 0.0
PP MMD Credit Default DICE Linear 0.0
PP MMD Credit Default DICE MLP 0.0
PP MMD Credit Default Generic (γ=0.5) Deep Ensemble 0.0
PP MMD Credit Default Generic (γ=0.5) Linear 0.0
PP MMD Credit Default Generic (γ=0.5) MLP 0.0
PP MMD Credit Default Greedy Deep Ensemble 0.0
PP MMD Credit Default Greedy Linear 0.044
PP MMD Credit Default Greedy MLP 0.0
PP MMD Credit Default Latent Deep Ensemble 0.0
PP MMD Credit Default Latent Linear 0.436
PP MMD Credit Default Latent MLP 0.0
PP MMD GMSC DICE Deep Ensemble 0.032
PP MMD GMSC DICE Linear 0.0
PP MMD GMSC DICE MLP 0.0
PP MMD GMSC Generic (γ=0.5) Deep Ensemble 0.018
PP MMD GMSC Generic (γ=0.5) Linear 0.0
PP MMD GMSC Generic (γ=0.5) MLP 0.0
PP MMD GMSC Greedy Deep Ensemble 0.02
PP MMD GMSC Greedy Linear 0.0
PP MMD GMSC Greedy MLP 0.0
PP MMD GMSC Latent Deep Ensemble 0.008
PP MMD GMSC Latent Linear 0.0
PP MMD GMSC Latent MLP 0.0
PP MMD (grid) Cal Housing DICE Deep Ensemble 0.0
PP MMD (grid) Cal Housing DICE Linear 0.0
PP MMD (grid) Cal Housing DICE MLP 0.0
PP MMD (grid) Cal Housing Generic (γ=0.5) Deep Ensemble 0.0
PP MMD (grid) Cal Housing Generic (γ=0.5) Linear 0.0
PP MMD (grid) Cal Housing Generic (γ=0.5) MLP 0.004
PP MMD (grid) Cal Housing Greedy Deep Ensemble 0.0
PP MMD (grid) Cal Housing Greedy Linear 0.0
PP MMD (grid) Cal Housing Greedy MLP 0.0
PP MMD (grid) Cal Housing Latent Deep Ensemble 0.006
PP MMD (grid) Cal Housing Latent Linear 0.01
PP MMD (grid) Cal Housing Latent MLP 0.026
PP MMD (grid) Credit Default DICE Deep Ensemble 0.0
PP MMD (grid) Credit Default DICE Linear 0.0
PP MMD (grid) Credit Default DICE MLP 0.0
PP MMD (grid) Credit Default Generic (γ=0.5) Deep Ensemble 0.0
PP MMD (grid) Credit Default Generic (γ=0.5) Linear 0.0
PP MMD (grid) Credit Default Generic (γ=0.5) MLP 0.0
PP MMD (grid) Credit Default Greedy Deep Ensemble 0.164
PP MMD (grid) Credit Default Greedy Linear 0.0
PP MMD (grid) Credit Default Greedy MLP 0.0
PP MMD (grid) Credit Default Latent Deep Ensemble 0.0
PP MMD (grid) Credit Default Latent Linear 0.044
PP MMD (grid) Credit Default Latent MLP 0.0
PP MMD (grid) GMSC DICE Deep Ensemble 0.0
PP MMD (grid) GMSC DICE Linear 0.0
PP MMD (grid) GMSC DICE MLP 0.004
PP MMD (grid) GMSC Generic (γ=0.5) Deep Ensemble 0.002
PP MMD (grid) GMSC Generic (γ=0.5) Linear 0.0
PP MMD (grid) GMSC Generic (γ=0.5) MLP 0.0
PP MMD (grid) GMSC Greedy Deep Ensemble 0.0
PP MMD (grid) GMSC Greedy Linear 0.0
PP MMD (grid) GMSC Greedy MLP 0.0
PP MMD (grid) GMSC Latent Deep Ensemble 0.0
PP MMD (grid) GMSC Latent Linear 0.0
PP MMD (grid) GMSC Latent MLP 0.03

4.2.5 Chart in paper (Figure 4)

Figure 4.7 shows the chart that went into the paper.

Code 
Images.load(joinpath(www_artifact_path,"paper_real_world_results.png"))
Figure 4.7: Chart in paper