How to choose training- Validation

[EYErbil]

Hello! I have created a script for it to go through every calculation I have did of a structure, and write me an input file, script below:

import argparse
from pathlib import Path
import os
import re
import logging
import json

def setup_logger(logfile):
    logging.basicConfig(
        filename=logfile,
        filemode="a",
        format="%(message)s",
        datefmt="%H:%M:%S",
        level=logging.DEBUG,
        force=True,
    )

def log(*args, sep="\t"):
    logger = logging.getLogger("setuplogger")
    info = sep.join([str(j) for j in args])
    logger.info(info)

def process_directories(base_path, pattern):
    directories = [p for p in base_path.iterdir() if p.is_dir() and pattern.search(p.name)]

    logfile = base_path.joinpath("setup.log")
    setup_logger(logfile)
    logger = logging.getLogger('setuplogger')

    training_dirs = []
    validation_dirs = []

    for dir_path in directories:
        output_file = dir_path / "output.out"
        if output_file.exists():
            data_dir = dir_path / "data"
            os.system(f"dpdata {output_file} -i cp2k/output -o deepmd/npy -O {data_dir}")

            # Add the data directory itself instead of the subdirectories
            if "validate" in dir_path.name.lower():
                validation_dirs.append(str(data_dir))
            else:
                training_dirs.append(str(data_dir))
            logger.info(f"Processed directory: {dir_path}")

    return training_dirs, validation_dirs

def generate_input_json(run_path, run, training_dirs, validation_dirs):
    input_json = {
        "_comment": " model parameters",
        "model": {
            "type_map": ["Mo", "S","H"],
            "descriptor": {
                "type": "se_e2_a", # atten and se_e3 give memory problems
                "sel": [500, 500],
                "rcut_smth": 2.0,
                "rcut": 8.00,
                "neuron": [25, 50, 100],
                "resnet_dt": False,
                "axis_neuron": 12, # only for se_e2_a
                "seed": 1,
                "_comment": " that's all"
            },
            "fitting_net": {
                "type": "ener",
                "neuron": [40, 40, 40],
                "resnet_dt": True,
                "seed": 1,
                "atom_ener": [-0.518285968665213E+06, -0.665434947249878E+05],
                "_comment": " that's all"
            },
            "_comment": " that's all"
        },
        "learning_rate": {
            "type": "exp",
            "decay_steps": 5000,
            "start_lr": 0.0001,
            "stop_lr": 3.51e-8,
            "_comment": "that's all"
        },
        "loss": {
            "type": "ener",
            "start_pref_e": 1,
            "limit_pref_e": 1,
            "start_pref_f": 100,
            "limit_pref_f": 1,
            "start_pref_v": 1,
            "limit_pref_v": 1,
            "_comment": " that's all"
        },
        "training": {
            "training_data": {
                "systems": training_dirs,
                "auto_prob": "prob_sys_size", #"prob_uniform",
                "batch_size": "1", #was auto 64
                "_comment": "that's all"
            },
            "validation_data": {
                "systems": validation_dirs,
                "auto_prob": "prob_sys_size", #"prob_uniform",
                "batch_size": "1", #was auto:64
                "numb_btch": 5,
                "_comment": "that's all"
            },
            "numb_steps": int(2.5e6),
            "seed": 10,
            "disp_file": "lcurve.out",
            "disp_freq": 1000,
            "save_freq": 50000,
            "_comment": "that's all"
        },
        "_comment": "cp2k"
    }

    with open(os.path.join(run_path, "input.json"), "w") as outfile:
        json.dump(input_json, outfile, indent=4)

def main():
    parser = argparse.ArgumentParser(description='A script for machine learning calculations.')
    parser.add_argument('path', type=str, help='The path to the files')
    parser.add_argument('run', type=str, help='The run identifier')
    args = parser.parse_args()

    base_path = Path(args.path)
    run = args.run

    # Define regex pattern for directory matching (e.g., directories starting with "3R")
    pattern = re.compile(r"3R")

    training_dirs, validation_dirs = process_directories(base_path, pattern)

    log("Validation systems")
    log("-----\n")
    for dir in validation_dirs:
        log(dir)

    log("Training systems")
    log("-----\n")
    for dir in training_dirs:
        log(dir)

    run_path = os.path.join(base_path, f"{run}.train")
    os.makedirs(run_path, exist_ok=True)
    
    generate_input_json(run_path, run, training_dirs, validation_dirs)

if __name__ == "__main__":
    main()

and the resulting input file is :
input.json

Now here, 2 problems:

1. the atom_ener has 2 potentials, which I think since my structures have 3 atoms, it should be 3, but I am not sure If I need to do this manually or deepmd does it auto.
2. No validation data! I know I can manually put some of the training there, but in a cluster with 30000 data, and randomly carrying stuff to validation would be a bad idea, any clever way to tackle this problem?

Also, my goal is to create a model to ease the DFT calculations, so if anyone has an input.json, that works well, please share for me to see and take examples

[wanghan-iapcm]

1. deepmd will automatically do linear regression over the training data to decide the atomic energy.
2. see this PR https://github.com/deepmodeling/dpdata/pull/459/files# and find example here https://github.com/njzjz/dpdata/blob/9d56937e85a7ed0ad904ecb2e96094f709f42e24/tests/test_split_dataset.py#L15-L21

[EYErbil]

I have one more questions, what I do is, I have many output files in a directory, my script goes in one by one to these files and converts the data, then rights the input.json, what I need is to seperate lets say 10000 calculation files as 0.8-0.2, but I couldnt find how I can do that, checked the link you sent, but I think this is for single output file, and uses to seperate frames.

[EYErbil]

I meant, I only run dpdata os.system(f"dpdata {output_file} -i cp2k/output -o deepmd/npy -O {data_dir}")
where can I include the validation and training sets?
my whole script is:

import argparse
from pathlib import Path
import os
import re
import logging
import json

def setup_logger(logfile):
    logging.basicConfig(
        filename=logfile,
        filemode="a",
        format="%(message)s",
        datefmt="%H:%M:%S",
        level=logging.DEBUG,
        force=True,
    )

def log(*args, sep="\t"):
    logger = logging.getLogger("setuplogger")
    info = sep.join([str(j) for j in args])
    logger.info(info)

def process_directories(base_path, pattern):
    directories = [p for p in base_path.iterdir() if p.is_dir() and pattern.search(p.name)]

    logfile = base_path.joinpath("setup.log")
    setup_logger(logfile)
    logger = logging.getLogger('setuplogger')

    training_dirs = []
    validation_dirs = []

    for dir_path in directories:
        output_file = dir_path / "output.out"
        if output_file.exists():
            data_dir = dir_path / "data"
            os.system(f"dpdata {output_file} -i cp2k/output -o deepmd/npy -O {data_dir}")

            # Add the data directory itself instead of the subdirectories
            if "validate" in dir_path.name.lower():
                validation_dirs.append(str(data_dir))
            else:
                training_dirs.append(str(data_dir))
            logger.info(f"Processed directory: {dir_path}")

    return training_dirs, validation_dirs

def generate_input_json(run_path, run, training_dirs, validation_dirs):
    input_json = {
        "_comment": " model parameters",
        "model": {
            "type_map": ["Mo", "S","H"],
            "descriptor": {
                "type": "se_e2_a", # atten and se_e3 give memory problems
                "sel": [500, 500],
                "rcut_smth": 2.0,
                "rcut": 8.00,
                "neuron": [25, 50, 100],
                "resnet_dt": False,
                "axis_neuron": 12, # only for se_e2_a
                "seed": 1,
                "_comment": " that's all"
            },
            "fitting_net": {
                "type": "ener",
                "neuron": [40, 40, 40],
                "resnet_dt": True,
                "seed": 1,
                "_comment": " that's all"
            },
            "_comment": " that's all"
        },
        "learning_rate": {
            "type": "exp",
            "decay_steps": 5000,
            "start_lr": 0.0001,
            "stop_lr": 3.51e-8,
            "_comment": "that's all"
        },
        "loss": {
            "type": "ener",
            "start_pref_e": 1,
            "limit_pref_e": 1,
            "start_pref_f": 100,
            "limit_pref_f": 1,
            "start_pref_v": 1,
            "limit_pref_v": 1,
            "_comment": " that's all"
        },
        "training": {
            "training_data": {
                "systems": training_dirs,
                "auto_prob": "prob_sys_size", #"prob_uniform",
                "batch_size": "1", #was auto 64
                "_comment": "that's all"
            },
            "validation_data": {
                "systems": validation_dirs,
                "auto_prob": "prob_sys_size", #"prob_uniform",
                "batch_size": "1", #was auto:64
                "numb_btch": 5,
                "_comment": "that's all"
            },
            "numb_steps": int(2.5e6),
            "seed": 10,
            "disp_file": "lcurve.out",
            "disp_freq": 1000,
            "save_freq": 50000,
            "_comment": "that's all"
        },
        "_comment": "cp2k"
    }

    with open(os.path.join(run_path, "input.json"), "w") as outfile:
        json.dump(input_json, outfile, indent=4)

def main():
    parser = argparse.ArgumentParser(description='A script for machine learning calculations.')
    parser.add_argument('path', type=str, help='The path to the files')
    parser.add_argument('run', type=str, help='The run identifier')
    args = parser.parse_args()

    base_path = Path(args.path)
    run = args.run

    # Define regex pattern for directory matching (e.g., directories starting with "3R")
    pattern = re.compile(r".*")

    training_dirs, validation_dirs = process_directories(base_path, pattern)

    log("Validation systems")
    log("-----\n")
    for dir in validation_dirs:
        log(dir)

    log("Training systems")
    log("-----\n")
    for dir in training_dirs:
        log(dir)

    run_path = os.path.join(base_path, f"{run}.train")
    os.makedirs(run_path, exist_ok=True)
    
    generate_input_json(run_path, run, training_dirs, validation_dirs)

if __name__ == "__main__":
    main()