"""A9: layered vegetation scattering against independent polarized adding/doubling."""
from pathlib import Path
import argparse
import importlib.util
import json
import time
import numpy as np
import less
from less.distributions import Fixed, DeWit
import appendix_microwave_brightness as ground


def load_reference():
    path = Path(__file__).with_name("microwave_layered_reference.py")
    if not path.exists():
        path = Path(__file__).resolve().parents[4]/"tests/reference/microwave_layered.py"
    spec = importlib.util.spec_from_file_location("microwave_layered_reference", path)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


REFERENCE = load_reference()


def references(config, groups):
    settings = config["scattering"]
    angles, frequency = settings["view_angles_degrees"], settings["frequency_Hz"]
    results = {}
    for group in groups:
        soil = ground.reference(config, dict(soil_moisture=settings["soil_moisture"],
            view_zenith_degrees=0., soil_K=group["soil_K"], tau=0., air_K=290.))
        eps = complex(soil["permittivity"][0], -soil["permittivity"][1])
        args = (group["layers"], frequency, eps, group["soil_K"], config["background_K"], angles)
        levels = [REFERENCE.solve_layers(*args, n_mu=n) for n in settings["reference_ordinates"]]
        n = settings["reference_ordinates"][-1]
        orders = REFERENCE.scattering_orders(*args, n_mu=n)
        thin = REFERENCE.solve_layers(*args, n_mu=n, thin_limit=.05)
        common = [dict(layer, temperature_K=300.) for layer in group["layers"]]
        equilibrium = REFERENCE.solve_layers(common, frequency, eps, 300., 300., angles, n_mu=n)
        rates = []
        for layer in group["layers"]:
            populations = [REFERENCE.Population(p, frequency, layer["temperature_K"])
                           for p in layer["populations"]]
            absorption = sum(p.rates(np.cos(np.deg2rad(angles)))[0] for p in populations)
            scattering = sum(p.rates(np.cos(np.deg2rad(angles)))[1] for p in populations)
            rates.append(dict(absorption_VH_m1=absorption.tolist(),
                scattering_VH_m1=scattering.tolist(),
                single_scattering_albedo_VH=(scattering/(absorption+scattering)).tolist(),
                vertical_extinction_VH=((absorption[0]+scattering[0])*
                    (layer["top_m"]-layer["bottom_m"])).tolist()))
        errors = dict(angular_K=float(np.max(abs(levels[-1]-levels[-2]))),
                      doubling_K=float(np.max(abs(thin-orders["full"]))),
                      equilibrium_K=float(np.max(abs(equilibrium.sum(axis=-1)-300.))))
        limits = settings["thresholds"]
        passed = all(errors[name] <= limits["reference_"+name] for name in errors)
        results[group["name"]] = dict(sources_VH_K=orders["full"].tolist(),
            brightness_VH_K=orders["full"].sum(axis=-1).tolist(),
            orders_VH_K={name: value.sum(axis=-1).tolist() for name, value in orders.items()},
            rates=rates, convergence=[dict(ordinates=n, brightness_VH_K=x.sum(axis=-1).tolist())
                for n, x in zip(settings["reference_ordinates"], levels)],
            errors=errors, passed=passed)
        print("reference", group["name"], errors, "higher K", orders["higher"].sum(axis=-1), flush=True)
    return results


def scene_for(config, group, backend):
    settings = config["scattering"]
    scene = ground.scene_for(config, dict(soil_moisture=settings["soil_moisture"],
        soil_K=group["soil_K"], air_K=290., tau=0.), backend)
    scene.measurement_plane = 0
    for index, layer in enumerate(group["layers"]):
        name = "layer_"+str(index)
        obj = less.Object(name)
        obj.add_component(name, less.TurbidBoundary(str(ground.DATA/"layer.obj")))
        populations = []
        for p in layer["populations"]:
            constructor = less.MicrowaveLeaf if p["shape"] == "leaf" else less.MicrowaveNeedle
            size = {"thickness" if p["shape"] == "leaf" else "length": p["length_m"]}
            populations.append(constructor(radius=p["radius_m"], **size,
                density=p["density_m3"], moisture=p["moisture"], dry_density=p["dry_density"],
                temperature=layer["temperature_K"], scattering_model="rayleigh",
                orientation=DeWit("spherical") if p["orientation"] == "spherical"
                    else Fixed(p["orientation"])))
        obj.set_property(name, less.MicrowaveVegetation(components=populations))
        count = round((layer["top_m"]-layer["bottom_m"])/2)
        assert count > 0 and abs(2*count-layer["top_m"]+layer["bottom_m"]) < 1e-12
        scene.add(obj, positions=[(0, 0, layer["bottom_m"]-1+2*i) for i in range(count)])
    return scene


def execute(config, group, scene, angle, quality, seed, angular=None):
    settings = config["scattering"]
    scene.sampler = less.Sampler("sobol", seed=seed)
    start = time.perf_counter()
    product = scene.simulate(less.MicrowaveRadiometer(
        geometry=less.Orthographic(image_size=settings["image_size"],
            extent=config["footprint_m"], altitude=config["sensor_altitude_m"],
            view_zenith=angle, view_azimuth=0.), frequency=settings["frequency_Hz"],
        polarization="full", quality=quality, seed=seed,
        scattering_samples=settings["angular_samples"] if angular is None else angular,
        atmosphere_optical_depth=0., volume_scattering=True))
    elapsed = time.perf_counter()-start
    identity = product.metadata["execution"]
    assert identity["native"] and identity["backend"] == scene.backend
    assert identity["executor"] == f"{scene.backend}.slang.microwave_radiometer"
    assert identity["workflow"] == "passive_microwave"
    assert product.metadata["volume_scattering"] is True
    image = np.stack((product.Tb_V, product.Tb_H), axis=-1).astype(float)
    sources = {name: np.asarray(product.sources["surface" if name == "soil" else name], float)[..., :2]
               for name in ("soil", "crown", "sky")}
    result = dict(seed=seed, brightness_VH_K=image.mean(axis=(0, 1)).tolist(),
        sources_VH_K=np.stack([sources[x].mean(axis=(0, 1))
            for x in ("soil", "crown", "sky")], axis=-1).tolist(),
        image_variance_VH_K2=image.var(axis=(0, 1)).tolist(),
        image_min_VH_K=image.min(axis=(0, 1)).tolist(), image_max_VH_K=image.max(axis=(0, 1)).tolist(),
        source_sum_K=float(np.max(abs(sum(sources.values())-image))),
        cross_stokes_mean_K=float(max(abs(product.Stokes_U.mean()), abs(product.Stokes_V.mean()))),
        wall_seconds=elapsed,
        execution={k: identity[k] for k in ("backend", "workflow", "executor", "native")})
    if group["name"] == "equilibrium":
        response = sources["soil"]/group["soil_K"]+sources["crown"]/300.+sources["sky"]/config["background_K"]
        result["thermal_response_mean_VH"] = response.mean(axis=(0, 1)).tolist()
        result["thermal_response_max_deviation"] = float(np.max(abs(response-1)))
    return result


def aggregate(replicas, exact):
    values = np.array([r["brightness_VH_K"] for r in replicas])
    difference = values-exact
    sources = np.array([r["sources_VH_K"] for r in replicas])
    minimum = np.array([r["image_min_VH_K"] for r in replicas])
    maximum = np.array([r["image_max_VH_K"] for r in replicas])
    return dict(replicas=replicas, brightness_VH_K=values.mean(axis=0).tolist(),
        sources_VH_K=sources.mean(axis=0).tolist(),
        standard_deviation_VH_K=values.std(axis=0, ddof=1).tolist() if len(replicas)>1 else [0., 0.],
        rmse_K=float(np.sqrt(np.mean(difference**2))),
        errors=dict(brightness_mean_K=float(np.max(abs(difference))),
            image_rmse_K=float(np.sqrt(np.mean(np.array([r["image_variance_VH_K2"] for r in replicas])+difference**2))),
            image_max_K=float(max(np.max(abs(minimum-exact)), np.max(abs(maximum-exact)))),
            source_sum_K=max(r["source_sum_K"] for r in replicas),
            stokes_cross_mean_K=max(r["cross_stokes_mean_K"] for r in replicas),
            seed_standard_deviation_K=float(np.max(values.std(axis=0, ddof=1))) if len(replicas)>1 else 0.))


def run(config, backends=ground.BACKENDS, selected=None):
    settings = config["scattering"]
    groups = [g for g in settings["groups"] if selected is None or g["name"] == selected]
    refs = references(config, groups)
    records, sampling, angular_checks = [], [], []
    for backend in backends:
        for group in groups:
            scene = scene_for(config, group, backend)
            ref = refs[group["name"]]
            for i, angle in enumerate(settings["view_angles_degrees"]):
                exact = np.array(ref["brightness_VH_K"])[i]
                levels = []
                budgets = settings["quality_levels"]
                seeds = settings["seeds"]
                for quality in budgets:
                    replicas = [execute(config, group, scene, angle, quality, seed) for seed in seeds]
                    row = aggregate(replicas, exact)
                    row["quality"] = quality
                    row["errors"]["source_mean_K"] = float(np.max(abs(
                        np.array([r["sources_VH_K"] for r in replicas])-np.array(ref["sources_VH_K"])[i])))
                    if group["name"] == "equilibrium":
                        responses = np.array([r["thermal_response_mean_VH"] for r in replicas])
                        mean = responses.mean(axis=0)
                        # Independent randomized runs determine uncertainty;
                        # the deterministic absorption-case tolerance is not
                        # applied to individual stochastic pixel estimates.
                        half = (9.92484320091807*responses.std(axis=0, ddof=1)/np.sqrt(3)
                                if len(responses) == 3 else np.zeros(2))
                        row["thermal_equilibrium"] = dict(mean_VH_K=(300*mean).tolist(),
                            confidence_99_half_width_VH_K=(300*half).tolist(),
                            maximum_mean_error_K=float(np.max(abs(300*mean-300.))))
                        row["errors"]["thermal_response_closure"] = float(
                            np.max(np.maximum(0., abs(mean-1)-half)))
                    levels.append(row)
                    print(backend, group["name"], angle, quality, row["errors"], flush=True)
                row = dict(levels[-1], backend=backend, group=group["name"], angle_degrees=angle)
                if len(levels)>1:
                    row["errors"]["sampling_mean_change_K"] = float(np.max(abs(
                        np.array(levels[-1]["brightness_VH_K"])-levels[-2]["brightness_VH_K"])))
                row["passed"] = all(value <= settings["thresholds"][key] for key, value in row["errors"].items())
                if "thermal_equilibrium" in row:
                    row["passed"] &= (row["thermal_equilibrium"]["maximum_mean_error_K"]
                        <= settings["thresholds"]["brightness_mean_K"])
                records.append(row)
                sampling.append(dict(backend=backend, group=group["name"], angle_degrees=angle,
                    levels=[{key: value for key, value in level.items() if key != "replicas"}
                            for level in levels]))
                if group["name"] == "layered_scattering" and angle == 60:
                    angular = [dict(samples=n, **execute(config, group, scene, angle,
                        budgets[-1], seeds[0], n)) for n in settings["angular_levels"]]
                    delta = float(np.max(abs(np.array(angular[-1]["brightness_VH_K"])-angular[-2]["brightness_VH_K"])))
                    angular_checks.append(dict(backend=backend, levels=angular, change_K=delta,
                        passed=delta <= settings["thresholds"]["angular_mean_change_K"]))
            scene._backend_runtime.close()
    signal = min(float(np.min(ref["orders_VH_K"]["higher"])) for ref in refs.values())
    result = dict(reference_method="Independent polarized discrete ordinates, matrix exponential and adding/doubling",
        references=refs, records=records, sampling=sampling, angular_checks=angular_checks,
        minimum_higher_order_signal_K=signal,
        passed=all(r["passed"] for r in records+angular_checks+list(refs.values()))
            and signal >= settings["thresholds"]["minimum_higher_order_K"])
    result["backend_parity"] = backend_parity(config, records)
    result["passed"] &= all(row["passed"] for row in result["backend_parity"])
    return result


def backend_parity(config, records):
    comparisons = []
    for reference in records:
        if reference["backend"] != "embree":
            continue
        for row in records:
            if (row["backend"] == "embree" or row["group"] != reference["group"]
                    or row["angle_degrees"] != reference["angle_degrees"]):
                continue
            delta = (np.array([r["brightness_VH_K"] for r in row["replicas"]])
                     -np.array([r["brightness_VH_K"] for r in reference["replicas"]]))
            mean = delta.mean(axis=0)
            half = (9.92484320091807*delta.std(axis=0, ddof=1)/np.sqrt(3)
                    if len(delta) == 3 else np.zeros(2))
            comparisons.append(dict(group=row["group"], angle_degrees=row["angle_degrees"],
                backend=row["backend"], reference_backend="embree",
                paired_mean_difference_VH_K=mean.tolist(),
                confidence_99_half_width_VH_K=half.tolist(),
                passed=bool(np.max(abs(mean)) <= config["scattering"]["thresholds"]["brightness_mean_K"]
                    and np.all(abs(mean) <= half+config["scattering"]["thresholds"]["source_sum_K"]))))
    return comparisons


def figures(config, result, output):
    import matplotlib
    matplotlib.use("Agg")
    import matplotlib.pyplot as plt
    plt.rcParams.update({"svg.fonttype": "none", "font.size": 10,
                         "axes.spines.top": False, "axes.spines.right": False})
    settings = config["scattering"]
    angles = settings["view_angles_degrees"]
    colors, markers = ("#2563a6", "#d47713", "#27976b"), ("o", "s", "^")
    fig, axes = plt.subplots(2, 2, figsize=(12, 8), layout="constrained")
    for panel, group, title in zip(axes[0], ("uniform_scattering", "layered_scattering"),
            ("(a) Uniform scattering layer", "(b) Three-layer canopy")):
        exact = np.array(result["references"][group]["brightness_VH_K"])
        for p, style in ((0, "-"), (1, "--")):
            panel.plot(angles, exact[:, p], style, color="#374151",
                       label=("V" if p == 0 else "H")+" reference")
            for backend, color, marker in zip(ground.BACKENDS, colors, markers):
                rows = sorted([r for r in result["records"] if r["group"] == group
                    and r["backend"] == backend], key=lambda r: r["angle_degrees"])
                panel.plot([r["angle_degrees"] for r in rows],
                    [r["brightness_VH_K"][p] for r in rows], linestyle="none",
                    color=color, marker=marker, markerfacecolor="none",
                    label=backend if p == 0 else None)
        panel.set(title=title, xlabel="View zenith / degrees", ylabel="Brightness temperature / K")
    axes[0, 0].legend(frameon=False, fontsize=8, ncols=2)
    first, higher, labels = [], [], []
    for group, short in (("uniform_scattering", "Uniform"), ("layered_scattering", "Layered")):
        orders = result["references"][group]["orders_VH_K"]
        for index, angle in enumerate(angles):
            if angle == 0:
                continue
            for p, name in enumerate(("V", "H")):
                first.append(orders["first"][index][p])
                higher.append(orders["higher"][index][p])
                labels.append(f"{short}\n{angle}° {name}")
    x = np.arange(len(first))
    axes[1, 0].bar(x, first, color="#7faed0", label="One volume scattering")
    axes[1, 0].bar(x, higher, bottom=first, color="#d47713", label="Two or more")
    axes[1, 0].set(title="(c) Scattered contribution: independent reference",
        ylabel="Brightness contribution / K", xticks=x, xticklabels=labels)
    axes[1, 0].tick_params(axis="x", labelsize=8)
    axes[1, 0].legend(frameon=False, fontsize=8)
    for backend, color, marker in zip(ground.BACKENDS, colors, markers):
        errors = []
        for quality in settings["quality_levels"]:
            errors.append(np.sqrt(np.mean([level["rmse_K"]**2
                for study in result["sampling"] if study["backend"] == backend
                for level in study["levels"] if level["quality"] == quality])))
        axes[1, 1].loglog(settings["quality_levels"], errors, color=color,
            marker=marker, markerfacecolor="none", label=backend)
    axes[1, 1].set(title="(d) Sampling convergence: three independent seeds",
                  xlabel="Samples per pixel", ylabel="Brightness-mean RMSE / K")
    axes[1, 1].legend(frameon=False, fontsize=8)
    for panel in axes.flat:
        panel.grid(alpha=.16, axis="y")
    fig.savefig(output/"scattering.svg")
    fig.savefig(output/"scattering.png", dpi=150)
    plt.close(fig)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--backend", choices=ground.BACKENDS)
    parser.add_argument("--group", choices=("uniform_scattering", "layered_scattering", "equilibrium"))
    parser.add_argument("--figures-only", action="store_true")
    parser.add_argument("--output", type=Path, default=Path("build/a9-scattering"))
    args = parser.parse_args()
    config = json.loads((ground.DATA/"input.json").read_text(encoding="utf-8"))
    args.output.mkdir(parents=True, exist_ok=True)
    if args.figures_only:
        result = json.loads((args.output/"results.json").read_text(encoding="utf-8"))
    else:
        result = run(config, [args.backend] if args.backend else ground.BACKENDS, args.group)
        (args.output/"results.json").write_text(json.dumps(result, indent=2)+"\n", encoding="utf-8")
    if args.backend is None and args.group is None:
        figures(config, result, args.output)
    raise SystemExit(0 if result["passed"] else 1)
