From cac0e468c73bd8309338e313d187b075f490b1e0 Mon Sep 17 00:00:00 2001 From: Roberto Lopez Date: Mon, 15 Jul 2024 20:57:04 +0200 Subject: [PATCH] Clean code --- opennn/batch.cpp | 6 +-- opennn/bounding_layer.cpp | 2 +- opennn/config.h | 8 ++-- opennn/convolutional_layer.cpp | 4 +- opennn/correlations.cpp | 4 +- opennn/cross_entropy_error.cpp | 6 --- opennn/data_set.cpp | 59 ++++++++++--------------- opennn/embedding_layer.cpp | 6 +-- opennn/flatten_layer.cpp | 6 +-- opennn/genetic_algorithm.cpp | 2 +- opennn/images.cpp | 18 ++++---- opennn/kmeans.h | 14 ------ opennn/language_data_set.cpp | 24 +++++++--- opennn/long_short_term_memory_layer.cpp | 8 ++-- opennn/loss_index.cpp | 51 ++++++++------------- opennn/mean_squared_error.cpp | 6 --- opennn/multihead_attention_layer.cpp | 24 +++++----- opennn/neural_network.cpp | 29 +++++------- opennn/neurons_selection.cpp | 2 + opennn/normalization_layer_3d.cpp | 1 - opennn/normalized_squared_error.cpp | 9 ---- opennn/opennn.pro | 2 +- opennn/perceptron_layer.cpp | 8 +--- opennn/perceptron_layer_3d.cpp | 6 +-- opennn/probabilistic_layer_3d.cpp | 56 +---------------------- opennn/quasi_newton_method.cpp | 5 --- opennn/recurrent_layer.cpp | 15 +++---- opennn/recurrent_layer.h | 2 - opennn/scaling_layer_2d.cpp | 6 +-- opennn/statistics.cpp | 45 ------------------- opennn/stochastic_gradient_descent.cpp | 2 +- opennn/strings_utilities.cpp | 49 +++----------------- opennn/strings_utilities.h | 4 -- opennn/sum_squared_error.cpp | 11 +---- opennn/tensors.h | 1 - opennn/testing_analysis.cpp | 8 ++-- opennn/testing_analysis.h | 2 +- opennn/tinyxml2.cpp | 2 +- opennn/tinyxml2.h | 4 +- opennn/transformer.cpp | 15 +++---- opennn/weighted_squared_error.cpp | 10 +---- opennn/weighted_squared_error.h | 2 +- 42 files changed, 159 insertions(+), 385 deletions(-) diff --git a/opennn/batch.cpp b/opennn/batch.cpp index c74a5bc9b..3191d91c3 100644 --- a/opennn/batch.cpp +++ b/opennn/batch.cpp @@ -80,7 +80,7 @@ void Batch::perform_augmentation() const if(random_rotation_minimum != 0 && random_rotation_maximum != 0) { - const type angle = (random_rotation_minimum < random_rotation_maximum) + const type angle = random_rotation_minimum < random_rotation_maximum ? random_rotation_minimum + type(rand()) : random_rotation_maximum; @@ -89,7 +89,7 @@ void Batch::perform_augmentation() const if(random_rescaling_minimum != 0 && random_rescaling_maximum != 0) { - const type rescaling = (random_rescaling_minimum < random_rescaling_maximum) + const type rescaling = random_rescaling_minimum < random_rescaling_maximum ? random_rescaling_minimum + type(rand()) : random_rescaling_maximum; @@ -98,7 +98,7 @@ void Batch::perform_augmentation() const if(random_horizontal_translation_minimum != 0 && random_horizontal_translation_maximum != 0) { - const type translation = (random_horizontal_translation_minimum < random_rescaling_maximum) + const type translation = random_horizontal_translation_minimum < random_rescaling_maximum ? random_horizontal_translation_minimum + type(rand()) : random_rescaling_maximum; diff --git a/opennn/bounding_layer.cpp b/opennn/bounding_layer.cpp index c166bf194..10e02b493 100644 --- a/opennn/bounding_layer.cpp +++ b/opennn/bounding_layer.cpp @@ -515,7 +515,7 @@ void BoundingLayer::write_XML(tinyxml2::XMLPrinter& file_stream) const { file_stream.OpenElement("Item"); - file_stream.PushAttribute("Index", static_cast(i+1)); + file_stream.PushAttribute("Index", unsigned(i+1)); // Lower bound diff --git a/opennn/config.h b/opennn/config.h index 6746c6610..ad8ade7d5 100644 --- a/opennn/config.h +++ b/opennn/config.h @@ -45,20 +45,20 @@ namespace opennn { - //using namespace std; + using namespace std; using namespace Eigen; using type = float; - using dimensions = std::vector; + using dimensions = vector; //using execution_policy = std::execution::par; template - inline bool is_instance_of(const T* ptr) { + inline bool is_instance_of(const T* ptr) + { return dynamic_cast(ptr) != nullptr; } - } diff --git a/opennn/convolutional_layer.cpp b/opennn/convolutional_layer.cpp index 5548b05f6..2bb0045bd 100644 --- a/opennn/convolutional_layer.cpp +++ b/opennn/convolutional_layer.cpp @@ -466,8 +466,8 @@ void ConvolutionalLayer::back_propagate(const Tensor, 1> Tensor& biases_derivatives = convolutional_layer_back_propagation->biases_derivatives; - Eigen::array offsets; - Eigen::array extents; + Eigen::array offsets; + Eigen::array extents; // Synaptic weights derivatives diff --git a/opennn/correlations.cpp b/opennn/correlations.cpp index ae800b858..74978c264 100644 --- a/opennn/correlations.cpp +++ b/opennn/correlations.cpp @@ -466,7 +466,7 @@ pair, Tensor> filter_missing_values_matrix_matrix(const Tensor get_correlation_values(const Tensor& correlations) { Tensor values; - /** + /* const Index rows_number = correlations.dimension(0); const Index raw_variables_number = correlations.dimension(1); Tensor values(rows_number, raw_variables_number); @@ -922,7 +922,7 @@ Correlation logistic_correlation_vector_vector_spearman(const ThreadPoolDevice* correlation.r = linear_correlation(thread_pool_device, outputs.reshape(vector), targets.reshape(vector)).r; const type z_correlation = r_correlation_to_z_correlation(correlation.r); -/** +/* const Tensor confidence_interval_z = confidence_interval_z_correlation(z_correlation, inputs_dimensions(0)); correlation.lower_confidence = z_correlation_to_r_correlation(confidence_interval_z(0)); diff --git a/opennn/cross_entropy_error.cpp b/opennn/cross_entropy_error.cpp index 981319212..000280aa5 100644 --- a/opennn/cross_entropy_error.cpp +++ b/opennn/cross_entropy_error.cpp @@ -167,9 +167,6 @@ void CrossEntropyError::calculate_binary_output_delta(const Batch& batch, const ProbabilisticLayerForwardPropagation* probabilistic_layer_forward_propagation = static_cast(forward_propagation.layers(last_trainable_layer_index)); - ProbabilisticLayerBackPropagation* probabilistic_layer_back_propagation - = static_cast(back_propagation.neural_network.layers(last_trainable_layer_index)); - const Tensor& outputs = probabilistic_layer_forward_propagation->outputs; // Back propagation @@ -189,9 +186,6 @@ void CrossEntropyError::calculate_multiple_output_delta(const Batch& batch, { const Index last_trainable_layer_index = neural_network->get_last_trainable_layer_index(); - ProbabilisticLayerBackPropagation* probabilistic_layer_back_propagation - = static_cast(back_propagation.neural_network.layers(last_trainable_layer_index)); - const Index batch_samples_number = batch.get_batch_samples_number(); const pair targets_pair = batch.get_targets_pair(); diff --git a/opennn/data_set.cpp b/opennn/data_set.cpp index d3842b8bd..aa203d731 100755 --- a/opennn/data_set.cpp +++ b/opennn/data_set.cpp @@ -68,7 +68,7 @@ DataSet::DataSet(const Tensor& data, const Index& samples_number, const void DataSet::set_default_columns_scalers() { - const Index columns_number = raw_variables.size(); + const Index raw_variables_number = raw_variables.size(); if (model_type == ModelType::ImageClassification) { @@ -76,7 +76,7 @@ void DataSet::set_default_columns_scalers() } else { - for (Index i = 0; i < columns_number; i++) + for (Index i = 0; i < raw_variables_number; i++) { if (raw_variables(i).type == RawVariableType::Numeric) { @@ -6343,7 +6343,7 @@ Tensor DataSet::calculate_input_target_raw_variables_correlation cout << "input_raw_variable_data: " << input_raw_variable_data << endl; cout << "target_raw_variable_data: " << target_raw_variable_data << endl; - correlations(i,j) = opennn::correlation(correlations_thread_pool_device, input_raw_variable_data, target_raw_variable_data); + correlations(i,j) = correlation(correlations_thread_pool_device, input_raw_variable_data, target_raw_variable_data); } } @@ -6366,8 +6366,7 @@ Tensor DataSet::calculate_relevant_input_target_raw_variables_co Tensor correlations(input_raw_variables_number, target_raw_variables_number); - -#pragma omp parallel for + #pragma omp parallel for for(Index i = 0; i < input_raw_variables_number; i++) { const Index input_index = input_raw_variables_indices(i); @@ -6379,17 +6378,17 @@ Tensor DataSet::calculate_relevant_input_target_raw_variables_co const Tensor input_raw_variable_data = get_raw_variable_data(input_index, get_used_samples_indices()); const Tensor target_raw_variable_data = get_raw_variable_data(target_index, get_used_samples_indices()); - correlations(i, j) = opennn::correlation(correlations_thread_pool_device, input_raw_variable_data, target_raw_variable_data); + correlations(i, j) = correlation(correlations_thread_pool_device, input_raw_variable_data, target_raw_variable_data); } } + delete correlations_thread_pool; delete correlations_thread_pool_device; + return correlations; } - - Tensor DataSet::calculate_input_target_raw_variables_correlations_spearman() const { const Index input_raw_variables_number = get_input_raw_variables_number(); @@ -6414,7 +6413,7 @@ Tensor DataSet::calculate_input_target_raw_variables_correlation const Tensor target_raw_variable_data = get_raw_variable_data(target_index, used_samples_indices); - correlations(i,j) = opennn::correlation_spearman(thread_pool_device, input_raw_variable_data, target_raw_variable_data); + correlations(i,j) = correlation_spearman(thread_pool_device, input_raw_variable_data, target_raw_variable_data); } } @@ -6596,27 +6595,17 @@ Tensor, 1> DataSet::calculate_input_raw_variables_correla if(calculate_pearson_correlations) { - correlations(i,j) = opennn::correlation(thread_pool_device, input_i, input_j); - - cout << "-=-=-=-=-=-" << endl; - cout << " i : " << i << "&&" << " j : " << j << endl; - cout << "-=-=-=-=-=-" << endl; - cout << "input_i :: " << input_i << endl; - cout << "input_j :: " << input_j << endl; - cout << "correlations(i,j).r :: " << correlations(i,j).r << endl; - cout << "type(1) :: " << type(1) << endl; - cout << "NUMERIC_LIMITS_MIN :: " << NUMERIC_LIMITS_MIN << endl; - cout << "-=-=-=-=-=-" << endl; - - if(correlations(i,j).r > (type(1) - NUMERIC_LIMITS_MIN)) + correlations(i,j) = correlation(thread_pool_device, input_i, input_j); + + if(correlations(i,j).r > type(1) - NUMERIC_LIMITS_MIN) correlations(i,j).r = type(1); } if(calculate_spearman_correlations) { - correlations_spearman(i,j) = opennn::correlation_spearman(thread_pool_device, input_i, input_j); + correlations_spearman(i,j) = correlation_spearman(thread_pool_device, input_i, input_j); - if(correlations_spearman(i,j).r > (type(1) - NUMERIC_LIMITS_MIN)) + if(correlations_spearman(i,j).r > type(1) - NUMERIC_LIMITS_MIN) correlations_spearman(i,j).r = type(1); } } @@ -7799,7 +7788,7 @@ void DataSet::from_XML(const tinyxml2::XMLDocument& data_set_document) } } - /** + /* // // Time series raw_variables // const tinyxml2::XMLElement* time_series_raw_variables_element = data_set_element->FirstChildElement("TimeSeriesraw_variables"); @@ -8461,7 +8450,7 @@ void DataSet::save_data() const ostringstream buffer; buffer << "OpenNN Exception: Matrix template." << endl - << "void save_csv(const string&, const char&, const Vector&, const Vector&) method." << endl + << "void save_data() method." << endl << "Cannot open matrix data file: " << data_source_path << endl; throw runtime_error(buffer.str()); @@ -9605,8 +9594,8 @@ void DataSet::read_csv_1() if(has_raw_variables_names) { - has_rows_labels ? set_raw_variables_names(data_file_preview(0).slice(Eigen::array({1}), - Eigen::array({data_file_preview(0).size()-1}))) + has_rows_labels ? set_raw_variables_names(data_file_preview(0).slice(Eigen::array({1}), + Eigen::array({data_file_preview(0).size()-1}))) : set_raw_variables_names(data_file_preview(0)); } else @@ -9857,6 +9846,7 @@ void DataSet::read_csv_2_simple() void DataSet::read_csv_3_simple() { const regex accent_regex("[\\xC0-\\xFF]"); + ifstream file; #ifdef _WIN32 @@ -9864,9 +9854,12 @@ void DataSet::read_csv_3_simple() if(std::regex_search(data_source_path, accent_regex)) { wstring_convert> conv; + wstring file_name_wide = conv.from_bytes(data_source_path); + file.open(file_name_wide); - }else + } + else { file.open(data_source_path.c_str()); } @@ -9964,8 +9957,6 @@ void DataSet::read_csv_3_simple() raw_variable_index = 0; sample_index++; - - type percentage = static_cast(sample_index)/static_cast(samples_number); } const Index data_file_preview_index = has_raw_variables_names ? 3 : 2; @@ -10080,7 +10071,7 @@ void DataSet::read_csv_2_complete() tokens_count = tokens.size(); - if(static_cast(tokens_count) != raw_raw_variables_number) + if(unsigned(tokens_count) != raw_raw_variables_number) { const string message = "Sample " + to_string(lines_count+1) + " error:\n" @@ -10131,7 +10122,7 @@ void DataSet::read_csv_2_complete() file.close(); - const Index samples_number = static_cast(lines_count); + const Index samples_number = unsigned(lines_count); const Index variables_number = get_variables_number(); @@ -10401,8 +10392,6 @@ void DataSet::read_csv_3_complete() } sample_index++; - - type percentage = static_cast(sample_index)/static_cast(data.dimension(0)); } const Index data_file_preview_index = has_raw_variables_names ? 3 : 2; diff --git a/opennn/embedding_layer.cpp b/opennn/embedding_layer.cpp index f1f092ff3..457888e4d 100644 --- a/opennn/embedding_layer.cpp +++ b/opennn/embedding_layer.cpp @@ -248,7 +248,7 @@ void EmbeddingLayer::set_parameters_random() { for(Index j = 0; j < embedding_weights.dimension(1); j++) { - const type random = static_cast(rand()/(RAND_MAX+1.0)); + const type random = type(rand()/(RAND_MAX+1.0)); embedding_weights(i, j) = minimum + (maximum - minimum)*random; } @@ -376,10 +376,6 @@ void EmbeddingLayer::back_propagate(const Tensor, 1>& in const TensorMap> deltas(deltas_pair(0).first, batch_samples_number, inputs_number, deltas_pair(0).second[2]); - // Forward propagation - - EmbeddingLayerForwardPropagation* embedding_layer_forward_propagation = static_cast(forward_propagation); - // Back propagation EmbeddingLayerBackPropagation* embedding_layer_back_propagation = static_cast(back_propagation); diff --git a/opennn/flatten_layer.cpp b/opennn/flatten_layer.cpp index 8e9480ef5..f96eb91a0 100644 --- a/opennn/flatten_layer.cpp +++ b/opennn/flatten_layer.cpp @@ -258,11 +258,11 @@ void FlattenLayer::from_XML(const tinyxml2::XMLDocument& document) const Index input_channels_number = Index(atoi(input_channels_number_element->GetText())); -// Tensor inputsDimensionTensor(4); + Tensor inputsDimensionTensor(4); -// inputsDimensionTensor.setValues({input_height, input_width, input_channels_number, 0}); + inputsDimensionTensor.setValues({input_height, input_width, input_channels_number, 0}); -// set(inputsDimensionTensor); + //set(inputsDimensionTensor); // @todo Change to dimensions /* diff --git a/opennn/genetic_algorithm.cpp b/opennn/genetic_algorithm.cpp index e6b97b918..26d0c8944 100644 --- a/opennn/genetic_algorithm.cpp +++ b/opennn/genetic_algorithm.cpp @@ -523,7 +523,7 @@ void GeneticAlgorithm::initialize_population_random() cout << "Creating initial random population" << endl; - srand(static_cast(time(nullptr))); + srand(unsigned(time(nullptr))); for(Index i = 0; i < individuals_number; i++) { diff --git a/opennn/images.cpp b/opennn/images.cpp index 06881f52a..fe040f87c 100644 --- a/opennn/images.cpp +++ b/opennn/images.cpp @@ -74,8 +74,8 @@ Tensor read_bmp_image(const string& filename) const Tensor data_without_padding = remove_padding(image_data, rows_number, columns_number, padding); - const Eigen::array dims_3D = {channels, rows_number, columns_number}; - const Eigen::array dims_1D = {rows_number*columns_number}; + const Eigen::array dims_3D = {channels, rows_number, columns_number}; + const Eigen::array dims_1D = {rows_number*columns_number}; Tensor red_channel_flatted = data_without_padding.reshape(dims_3D).chip(2,0).reshape(dims_1D); // row_major Tensor green_channel_flatted = data_without_padding.reshape(dims_3D).chip(1,0).reshape(dims_1D); // row_major @@ -124,9 +124,9 @@ ImageData read_bmp_image_gpt(const std::string& filename) char header[54]; file.read(header, 54); - int width = *(int*)&header[18]; - int height = *(int*)&header[22]; - int channels = header[28] == 24 ? 3 : 1; + const int width = *(int*)&header[18]; + const int height = *(int*)&header[22]; + const int channels = header[28] == 24 ? 3 : 1; int padding = 0; while ((channels * width + padding) % 4 != 0) @@ -134,7 +134,7 @@ ImageData read_bmp_image_gpt(const std::string& filename) padding++; } - size_t size = height * (channels * width + padding); + const size_t size = height * (channels * width + padding); Tensor image_data(size); file.seekg(54, ios::beg); @@ -148,8 +148,8 @@ ImageData read_bmp_image_gpt(const std::string& filename) const Tensor data_without_padding = remove_padding(image_data, rows_number, columns_number, padding); - const Eigen::array dims_3D = {channels, rows_number, columns_number}; - const Eigen::array dims_1D = {rows_number*columns_number}; + const Eigen::array dims_3D = {channels, rows_number, columns_number}; + const Eigen::array dims_1D = {rows_number*columns_number}; Tensor red_channel_flatted = data_without_padding.reshape(dims_3D).chip(2,0).reshape(dims_1D); Tensor green_channel_flatted = data_without_padding.reshape(dims_3D).chip(1,0).reshape(dims_1D); @@ -440,7 +440,7 @@ Tensor resize_image(Tensor &data, // memcpy(data.data(), image_data.data(), images_number * image_size * sizeof(type)); - copy(/*execution::par, + copy(execution::par, image_data.data(), image_data.data() + images_number * image_size, data.data()); rows_labels.resize(images_number); diff --git a/opennn/kmeans.h b/opennn/kmeans.h index 74a2bea70..9bd045199 100755 --- a/opennn/kmeans.h +++ b/opennn/kmeans.h @@ -30,20 +30,6 @@ // System includes -//#include -//#include -//#include -//#include -//#include -//#include "unsupported/Eigen/CXX11/Tensor" - -// OpenNN includes - -//#include "../../opennn/opennn/opennn.h" - -using Eigen::Tensor; -using opennn::type; -using Eigen::Index; using namespace std; namespace opennn diff --git a/opennn/language_data_set.cpp b/opennn/language_data_set.cpp index 53c1cd045..e5ca20108 100644 --- a/opennn/language_data_set.cpp +++ b/opennn/language_data_set.cpp @@ -1784,7 +1784,17 @@ const Tensor LanguageDataSet::calculate_vocabulary(const Tensor> word_counts = count_words(total_tokens); - WordpieceAlgorithmParameters parameters = { upper_threshold, lower_threshold, interations_number, max_input_tokens, max_token_length, max_unique_characters, vocabulary_size, slack_ratio, include_joiner_token, joiner, reserved_tokens }; + WordpieceAlgorithmParameters parameters = { upper_threshold, + lower_threshold, + interations_number, + max_input_tokens, + max_token_length, + max_unique_characters, + vocabulary_size, + slack_ratio, + include_joiner_token, + joiner, + reserved_tokens }; auto [upper_search, lower_search] = calculate_thresholds(word_counts, parameters.upper_threshold, parameters.lower_threshold); @@ -1797,7 +1807,9 @@ const Tensor LanguageDataSet::calculate_vocabulary(const Tensor vocabulary = calculate_vocabulary_binary_search(filtered_counts, lower_search, upper_search, parameters); Tensor vocabulary_tensor(vocabulary.size()); - for (Index i = 0; i < vocabulary.size(); i++) vocabulary_tensor(i) = vocabulary[i]; + + for (Index i = 0; i < vocabulary.size(); i++) + vocabulary_tensor(i) = vocabulary[i]; return vocabulary_tensor; } @@ -2348,8 +2360,8 @@ void LanguageDataSet::write_data_file_wordpiece(ofstream& file, unordered_map completion_vocabulary_map; for (Index i = 0; i < completion_vocabulary.size(); i++) completion_vocabulary_map[completion_vocabulary(i)] = type(i); - const Index context_vocabulary_size = context_vocabulary.size(); - const Index completion_vocabulary_size = completion_vocabulary.size(); +// const Index context_vocabulary_size = context_vocabulary.size(); +// const Index completion_vocabulary_size = completion_vocabulary.size(); Tensor context_row(max_context_length + 2); Tensor completion_row(max_completion_length + 2); @@ -2363,11 +2375,11 @@ void LanguageDataSet::write_data_file_wordpiece(ofstream& file, string rest; auto wordpiece_entry = context_vocabulary_map.find(""); + bool tokenized; for (Index i = 0; i < entry_number; i++) - { - + { // Context context_row.setZero(); diff --git a/opennn/long_short_term_memory_layer.cpp b/opennn/long_short_term_memory_layer.cpp index 981b96de2..f4b109ad6 100755 --- a/opennn/long_short_term_memory_layer.cpp +++ b/opennn/long_short_term_memory_layer.cpp @@ -1690,7 +1690,7 @@ void LongShortTermMemoryLayer::calculate_forget_parameters_derivatives(const Ten forget_combinations_weights_derivatives(i, weight_index) += current_input; forget_combinations_recurrent_weights_derivatives(i, recurrent_weight_index) += previous_hidden_state_activation; - forget_combinations_biases_derivatives(i, i) += static_cast(1.0); + forget_combinations_biases_derivatives(i, i) += type(1.0); input_index++; neuron_index++; @@ -2026,7 +2026,7 @@ void LongShortTermMemoryLayer::calculate_input_parameters_derivatives(const Tens input_combinations_weights_derivatives(i, weight_index) += current_input; input_combinations_recurrent_weights_derivatives(i, recurrent_weight_index) += previous_hidden_state_activation; - input_combinations_biases_derivatives(i, i) += static_cast(1.0); + input_combinations_biases_derivatives(i, i) += type(1.0); input_index++; neuron_index++; @@ -2363,7 +2363,7 @@ void LongShortTermMemoryLayer::calculate_state_parameters_derivatives(const Tens state_combinations_weights_derivatives(i, weight_index) += current_input; state_combinations_recurrent_weights_derivatives(i, recurrent_weight_index) += previous_hidden_state_activation; - state_combinations_biases_derivatives(i, i) += static_cast(1.0); + state_combinations_biases_derivatives(i, i) += type(1.0); input_index++; neuron_index++; @@ -2700,7 +2700,7 @@ void LongShortTermMemoryLayer::calculate_output_parameters_derivatives(const Ten output_combinations_weights_derivatives(i, weight_index) += current_input; output_combinations_recurrent_weights_derivatives(i, recurrent_weight_index) += previous_hidden_state_activation; - output_combinations_biases_derivatives(i, i) += static_cast(1.0); + output_combinations_biases_derivatives(i, i) += type(1.0); input_index++; neuron_index++; diff --git a/opennn/loss_index.cpp b/opennn/loss_index.cpp index 1c0e58750..37ee7286d 100644 --- a/opennn/loss_index.cpp +++ b/opennn/loss_index.cpp @@ -383,26 +383,22 @@ void LossIndex::back_propagate(const Batch& batch, void LossIndex::add_regularization(BackPropagation& back_propagation) const { - // Regularization + if (regularization_method == RegularizationMethod::NoRegularization) return; - if (regularization_method != RegularizationMethod::NoRegularization) - { - const type& error = back_propagation.error; - type& regularization = back_propagation.regularization; - type& loss = back_propagation.loss; + type& regularization = back_propagation.regularization; + type& loss = back_propagation.loss; - const Tensor& parameters = back_propagation.parameters; - Tensor& regularization_gradient = back_propagation.regularization_gradient; - Tensor& gradient = back_propagation.gradient; + const Tensor& parameters = back_propagation.parameters; + Tensor& regularization_gradient = back_propagation.regularization_gradient; + Tensor& gradient = back_propagation.gradient; - regularization = calculate_regularization(parameters); + regularization = calculate_regularization(parameters); - loss += regularization_weight * regularization; + loss += regularization_weight * regularization; - calculate_regularization_gradient(parameters, regularization_gradient); + calculate_regularization_gradient(parameters, regularization_gradient); - gradient.device(*thread_pool_device) += regularization_weight * regularization_gradient; - } + gradient.device(*thread_pool_device) += regularization_weight * regularization_gradient; } @@ -463,8 +459,7 @@ void LossIndex::back_propagate_lm(const Batch& batch, void LossIndex::calculate_layers_squared_errors_jacobian_lm(const Batch& batch, ForwardPropagation& forward_propagation, BackPropagationLM& back_propagation_lm) const -{ - +{ const Tensor layers = neural_network->get_layers(); const Index layers_number = layers.size(); @@ -484,10 +479,7 @@ void LossIndex::calculate_layers_squared_errors_jacobian_lm(const Batch& batch, Layer* layer = nullptr; LayerForwardPropagation* layer_forward_propagation = nullptr; - LayerForwardPropagation* previous_layer_forward_propagation = nullptr; LayerBackPropagationLM* layer_back_propagation = nullptr; - LayerForwardPropagation* next_layer_forward_propagation = nullptr; - LayerBackPropagationLM* next_layer_back_propagation = nullptr; Tensor, 1> layer_inputs; Tensor, 1> layer_deltas; @@ -498,8 +490,7 @@ void LossIndex::calculate_layers_squared_errors_jacobian_lm(const Batch& batch, calculate_output_delta_lm(batch, forward_propagation, back_propagation_lm); for (Index i = last_trainable_layer_index; i >= first_trainable_layer_index; i--) - { - + { layer = layers(i); layer_forward_propagation = forward_propagation.layers(i); @@ -542,8 +533,7 @@ void LossIndex::calculate_layers_squared_errors_jacobian_lm(const Batch& batch, } } - layer->back_propagate_lm(layer_inputs, layer_deltas, layer_forward_propagation, layer_back_propagation); - + layer->back_propagate_lm(layer_inputs, layer_deltas, layer_forward_propagation, layer_back_propagation); } Index memory_index = 0; @@ -558,7 +548,6 @@ void LossIndex::calculate_layers_squared_errors_jacobian_lm(const Batch& batch, memory_index += trainable_layers_parameters_number(i) * batch_samples_number; } - } @@ -703,10 +692,7 @@ void LossIndex::calculate_layers_error_gradient(const Batch& batch, Layer* layer = nullptr; LayerForwardPropagation* layer_forward_propagation = nullptr; - LayerForwardPropagation* previous_layer_forward_propagation = nullptr; LayerBackPropagation* layer_back_propagation = nullptr; - LayerForwardPropagation* next_layer_forward_propagation = nullptr; - LayerBackPropagation* next_layer_back_propagation = nullptr; Tensor, 1> layer_inputs; Tensor, 1> layer_deltas; @@ -728,9 +714,6 @@ void LossIndex::calculate_layers_error_gradient(const Batch& batch, layer_deltas.resize(1); layer_deltas(0) = back_propagation.get_output_deltas_pair(); - //cout << "probabilistic layer deltas pair dimensions : " << endl << layer_deltas(0).second[0] << " , " << layer_deltas(0).second[1] << endl; - //const TensorMap> test(layer_deltas(0).first, layer_deltas(0).second[0], layer_deltas(0).second[1]); - //cout << test << endl; } else { @@ -788,8 +771,8 @@ void LossIndex::assemble_layers_error_gradient(BackPropagation& back_propagation for(Index i = 0; i < layers_number; i++) { layers(i)->insert_gradient(back_propagation.neural_network.layers(i), - index, - back_propagation.gradient); + index, + back_propagation.gradient); index += layers_parameters_number(i); } @@ -974,9 +957,11 @@ void BackPropagation::set(const Index& new_batch_samples_number, LossIndex* new_ output_deltas_dimensions[0] = batch_samples_number; Index size = batch_samples_number; + for (Index i = 0; i < output_dimensions.size(); i++) { output_deltas_dimensions[i + 1] = output_dimensions[i]; + size *= output_dimensions[i]; } @@ -1305,9 +1290,11 @@ void BackPropagationLM::set(const Index &new_batch_samples_number, output_deltas_dimensions[0] = batch_samples_number; Index size = batch_samples_number; + for (Index i = 0; i < output_dimensions.size(); i++) { output_deltas_dimensions[i + 1] = output_dimensions[i]; + size *= output_dimensions[i]; } diff --git a/opennn/mean_squared_error.cpp b/opennn/mean_squared_error.cpp index 635276bb7..df4137498 100644 --- a/opennn/mean_squared_error.cpp +++ b/opennn/mean_squared_error.cpp @@ -134,14 +134,8 @@ void MeanSquaredError::calculate_output_delta_lm(const Batch&, ForwardPropagation&, BackPropagationLM& back_propagation) const { - // Neural network - - const Index last_trainable_layer_index = neural_network->get_last_trainable_layer_index(); - // Back propagation - LayerBackPropagationLM* output_layer_back_propagation = back_propagation.neural_network.layers(last_trainable_layer_index); - const Tensor& errors = back_propagation.errors; const Tensor& squared_errors = back_propagation.squared_errors; diff --git a/opennn/multihead_attention_layer.cpp b/opennn/multihead_attention_layer.cpp index 308d47917..3c522ff88 100644 --- a/opennn/multihead_attention_layer.cpp +++ b/opennn/multihead_attention_layer.cpp @@ -443,7 +443,7 @@ void MultiheadAttentionLayer::set_parameters_random() #pragma omp parallel for for(Index i = 0; i < query_weights.size(); i++) { - const type random = static_cast(rand()/(RAND_MAX+1.0)); + const type random = type(rand()/(RAND_MAX+1.0)); query_weights(i) = minimum + (maximum - minimum)*random; } @@ -451,7 +451,7 @@ void MultiheadAttentionLayer::set_parameters_random() #pragma omp parallel for for (Index i = 0; i < query_biases.size(); i++) { - const type random = static_cast(rand() / (RAND_MAX + 1.0)); + const type random = type(rand() / (RAND_MAX + 1.0)); query_biases(i) = minimum + (maximum - minimum) * random; } @@ -459,7 +459,7 @@ void MultiheadAttentionLayer::set_parameters_random() #pragma omp parallel for for(Index i = 0; i < key_weights.size(); i++) { - const type random = static_cast(rand()/(RAND_MAX+1.0)); + const type random = type(rand()/(RAND_MAX+1.0)); key_weights(i) = minimum + (maximum - minimum)*random; } @@ -467,7 +467,7 @@ void MultiheadAttentionLayer::set_parameters_random() #pragma omp parallel for for (Index i = 0; i < key_biases.size(); i++) { - const type random = static_cast(rand() / (RAND_MAX + 1.0)); + const type random = type(rand() / (RAND_MAX + 1.0)); key_biases(i) = minimum + (maximum - minimum) * random; } @@ -475,7 +475,7 @@ void MultiheadAttentionLayer::set_parameters_random() #pragma omp parallel for for(Index i = 0; i < value_weights.size(); i++) { - const type random = static_cast(rand()/(RAND_MAX+1.0)); + const type random = type(rand()/(RAND_MAX+1.0)); value_weights(i) = minimum + (maximum - minimum)*random; } @@ -483,7 +483,7 @@ void MultiheadAttentionLayer::set_parameters_random() #pragma omp parallel for for (Index i = 0; i < value_biases.size(); i++) { - const type random = static_cast(rand() / (RAND_MAX + 1.0)); + const type random = type(rand() / (RAND_MAX + 1.0)); value_biases(i) = minimum + (maximum - minimum) * random; } @@ -491,7 +491,7 @@ void MultiheadAttentionLayer::set_parameters_random() #pragma omp parallel for for(Index i = 0; i < projection_weights.size(); i++) { - const type random = static_cast(rand()/(RAND_MAX+1.0)); + const type random = type(rand()/(RAND_MAX+1.0)); projection_weights(i) = minimum + (maximum - minimum)*random; } @@ -499,7 +499,7 @@ void MultiheadAttentionLayer::set_parameters_random() #pragma omp parallel for for (Index i = 0; i < projection_biases.size(); i++) { - const type random = static_cast(rand() / (RAND_MAX + 1.0)); + const type random = type(rand() / (RAND_MAX + 1.0)); projection_biases(i) = minimum + (maximum - minimum) * random; } @@ -521,7 +521,7 @@ void MultiheadAttentionLayer::set_parameters_glorot() #pragma omp parallel for for(Index i = 0; i < query_weights.size(); i++) { - const type random = static_cast(rand()/(RAND_MAX+1.0)); + const type random = type(rand()/(RAND_MAX+1.0)); query_weights(i) = minimum + (maximum - minimum)*random; } @@ -529,7 +529,7 @@ void MultiheadAttentionLayer::set_parameters_glorot() #pragma omp parallel for for(Index i = 0; i < key_weights.size(); i++) { - const type random = static_cast(rand()/(RAND_MAX+1.0)); + const type random = type(rand()/(RAND_MAX+1.0)); key_weights(i) = minimum + (maximum - minimum)*random; } @@ -537,7 +537,7 @@ void MultiheadAttentionLayer::set_parameters_glorot() #pragma omp parallel for for(Index i = 0; i < value_weights.size(); i++) { - const type random = static_cast(rand()/(RAND_MAX+1.0)); + const type random = type(rand()/(RAND_MAX+1.0)); value_weights(i) = minimum + (maximum - minimum)*random; } @@ -545,7 +545,7 @@ void MultiheadAttentionLayer::set_parameters_glorot() #pragma omp parallel for for(Index i = 0; i < projection_weights.size(); i++) { - const type random = static_cast(rand()/(RAND_MAX+1.0)); + const type random = type(rand()/(RAND_MAX+1.0)); projection_weights(i) = minimum + (maximum - minimum)*random; } diff --git a/opennn/neural_network.cpp b/opennn/neural_network.cpp index b2b3e8b68..a9a2940b0 100644 --- a/opennn/neural_network.cpp +++ b/opennn/neural_network.cpp @@ -1253,33 +1253,30 @@ Index NeuralNetwork::get_inputs_number() const Index NeuralNetwork::get_outputs_number() const { - if(layers.size() > 0) - { - const Layer* last_layer = layers[layers.size() - 1]; + if(layers.size() == 0) return 0; - const dimensions outputs_dimensions = last_layer->get_outputs_dimensions(); + const Layer* last_layer = layers[layers.size() - 1]; - Index outputs_number = 1; + const dimensions outputs_dimensions = last_layer->get_outputs_dimensions(); - for (Index i = 0; i < outputs_dimensions.size(); i++) outputs_number *= outputs_dimensions[i]; + const Index outputs_rank = outputs_dimensions.size(); - return outputs_number; - } + Index outputs_number = 1; - return 0; + for (Index i = 0; i < outputs_rank; i++) + outputs_number *= outputs_dimensions[i]; + + return outputs_number; } dimensions NeuralNetwork::get_outputs_dimensions() const { - if (layers.size() > 0) - { - const Layer* last_layer = layers[layers.size() - 1]; + if (layers.size() == 0) return {}; - return last_layer->get_outputs_dimensions(); - } + const Layer* last_layer = layers[layers.size() - 1]; - return {}; + return last_layer->get_outputs_dimensions(); } @@ -1885,7 +1882,6 @@ Tensor NeuralNetwork::calculate_outputs(const Tensor& inputs) { const Index batch_samples_number = inputs.dimension(0); const Index inputs_number = inputs.dimension(1); - const Index outputs_number = get_outputs_number(); ForwardPropagation neural_network_forward_propagation(batch_samples_number, this); @@ -1908,7 +1904,6 @@ Tensor NeuralNetwork::calculate_outputs(const Tensor& inputs) Tensor NeuralNetwork::calculate_outputs(const Tensor& inputs) { const Index batch_samples_number = inputs.dimension(0); - const Index outputs_number = get_outputs_number(); ForwardPropagation neural_network_forward_propagation(batch_samples_number, this); diff --git a/opennn/neurons_selection.cpp b/opennn/neurons_selection.cpp index 4284ab9a6..4d3d4e578 100644 --- a/opennn/neurons_selection.cpp +++ b/opennn/neurons_selection.cpp @@ -519,6 +519,7 @@ NeuronsSelectionResults::NeuronsSelectionResults(const Index& maximum_epochs_num optimum_selection_error = numeric_limits::max(); } + void NeuronsSelectionResults::resize_history(const Index& new_size) { const Tensor old_neurons_number_history = neurons_number_history; @@ -537,6 +538,7 @@ void NeuronsSelectionResults::resize_history(const Index& new_size) } } + string NeuronsSelectionResults::write_stopping_condition() const { switch(stopping_condition) diff --git a/opennn/normalization_layer_3d.cpp b/opennn/normalization_layer_3d.cpp index 9ae2287f7..127364a05 100644 --- a/opennn/normalization_layer_3d.cpp +++ b/opennn/normalization_layer_3d.cpp @@ -334,7 +334,6 @@ void NormalizationLayer3D::back_propagate(const Tensor, const Tensor& normalized_inputs = normalization_layer_3d_forward_propagation->normalized_inputs; - const Tensor& means = normalization_layer_3d_forward_propagation->means; const Tensor& standard_deviations = normalization_layer_3d_forward_propagation->standard_deviations; const TensorMap> standard_deviations_matrix((type*)standard_deviations.data(), batch_samples_number, inputs_number); diff --git a/opennn/normalized_squared_error.cpp b/opennn/normalized_squared_error.cpp index e20d8ee59..5b8ad40ce 100644 --- a/opennn/normalized_squared_error.cpp +++ b/opennn/normalized_squared_error.cpp @@ -312,8 +312,6 @@ void NormalizedSquaredError::calculate_output_delta(const Batch& batch, const Index total_samples_number = data_set->get_samples_number(); - const Index last_trainable_layer_index = neural_network->get_last_trainable_layer_index(); - // Batch const Index batch_samples_number = batch.get_batch_samples_number(); @@ -322,8 +320,6 @@ void NormalizedSquaredError::calculate_output_delta(const Batch& batch, const Tensor& errors = back_propagation.errors; - LayerBackPropagation* output_layer_back_propagation = back_propagation.neural_network.layers(last_trainable_layer_index); - const pair deltas_pair = back_propagation.get_output_deltas_pair(); TensorMap> deltas(deltas_pair.first, deltas_pair.second[0], deltas_pair.second[1]); @@ -338,10 +334,6 @@ void NormalizedSquaredError::calculate_output_delta_lm(const Batch& , ForwardPropagation&, BackPropagationLM & back_propagation) const { - const Index trainable_layers_number = neural_network->get_trainable_layers_number(); - - LayerBackPropagationLM* output_layer_back_propagation = back_propagation.neural_network.layers(trainable_layers_number-1); - const Tensor& errors = back_propagation.errors; const Tensor& squared_errors = back_propagation.squared_errors; @@ -394,7 +386,6 @@ void NormalizedSquaredError::calculate_error_hessian_lm(const Batch& batch, // Back propagation - const Tensor& squared_errors = back_propagation_lm.squared_errors; const Tensor& squared_errors_jacobian = back_propagation_lm.squared_errors_jacobian; Tensor& hessian = back_propagation_lm.hessian; diff --git a/opennn/opennn.pro b/opennn/opennn.pro index 7b915f773..3cce45b2a 100644 --- a/opennn/opennn.pro +++ b/opennn/opennn.pro @@ -22,7 +22,7 @@ CONFIG(debug, debug|release) { DEFINES += __Cpp17__ -#QMAKE_CXXFLAGS += -bigobj +QMAKE_CXXFLAGS += -bigobj # OpenMP library diff --git a/opennn/perceptron_layer.cpp b/opennn/perceptron_layer.cpp index 0b75c3b51..bafee1dd9 100644 --- a/opennn/perceptron_layer.cpp +++ b/opennn/perceptron_layer.cpp @@ -460,12 +460,8 @@ void PerceptronLayer::calculate_combinations(const Tensor& inputs, } - void PerceptronLayer::dropout(Tensor& outputs) const -{ - type* outputs_data = outputs.data(); - - const Index batch_samples_number = outputs.dimension(0); +{ const Index outputs_number = outputs.dimension(1); const type scaling_factor = type(1) / (type(1) - dropout_rate); @@ -659,8 +655,6 @@ void PerceptronLayer::back_propagate_lm(const Tensor, 1> const Index synaptic_weights_number = get_synaptic_weights_number(); - const Index samples_number = inputs.dimension(0); - // Forward propagation const PerceptronLayerForwardPropagation* perceptron_layer_forward_propagation = diff --git a/opennn/perceptron_layer_3d.cpp b/opennn/perceptron_layer_3d.cpp index 49533e14b..7282b0d73 100644 --- a/opennn/perceptron_layer_3d.cpp +++ b/opennn/perceptron_layer_3d.cpp @@ -140,7 +140,7 @@ Tensor PerceptronLayer3D::get_synaptic_weights(const Tensor& p const Index start_synaptic_weights_number = (parameters_size - synaptic_weights_number); - const Tensor new_synaptic_weights = parameters.slice(Eigen::array({start_synaptic_weights_number}), Eigen::array({synaptic_weights_number})); + const Tensor new_synaptic_weights = parameters.slice(Eigen::array({start_synaptic_weights_number}), Eigen::array({synaptic_weights_number})); const Eigen::array two_dim{{inputs_depth, neurons_number}}; @@ -152,7 +152,7 @@ Tensor PerceptronLayer3D::get_biases(const Tensor& parameters) { const Index biases_number = biases.size(); - const Tensor new_biases = parameters.slice(Eigen::array({0}), Eigen::array({biases_number})); + const Tensor new_biases = parameters.slice(Eigen::array({0}), Eigen::array({biases_number})); const Eigen::array two_dim{{1, biases.dimension(1)}}; @@ -502,7 +502,7 @@ void PerceptronLayer3D::set_parameters_glorot() for (Index i = 0; i < synaptic_weights.size(); i++) { - const type random = static_cast(rand() / (RAND_MAX + 1.0)); + const type random = type(rand() / (RAND_MAX + 1.0)); synaptic_weights(i) = minimum + (maximum - minimum) * random; } diff --git a/opennn/probabilistic_layer_3d.cpp b/opennn/probabilistic_layer_3d.cpp index ea59b3ff1..1aa7e1658 100644 --- a/opennn/probabilistic_layer_3d.cpp +++ b/opennn/probabilistic_layer_3d.cpp @@ -350,8 +350,6 @@ void ProbabilisticLayer3D::set_default() layer_type = Layer::Type::Probabilistic3D; - const Index neurons_number = get_neurons_number(); - activation_function = ActivationFunction::Softmax; decision_threshold = type(0.5); @@ -366,56 +364,6 @@ void ProbabilisticLayer3D::set_default() void ProbabilisticLayer3D::set_activation_function(const ActivationFunction& new_activation_function) { -#ifdef OPENNN_DEBUG - - const Index neurons_number = get_neurons_number(); - - if(neurons_number == 1 && new_activation_function == ActivationFunction::Competitive) - { - ostringstream buffer; - - buffer << "OpenNN Exception: ProbabilisticLayer3D class.\n" - << "void set_activation_function(const ActivationFunction&) method.\n" - << "Activation function cannot be Competitive when the number of neurons is 1.\n"; - - throw runtime_error(buffer.str()); - } - - if(neurons_number == 1 && new_activation_function == ActivationFunction::Softmax) - { - ostringstream buffer; - - buffer << "OpenNN Exception: ProbabilisticLayer3D class.\n" - << "void set_activation_function(const ActivationFunction&) method.\n" - << "Activation function cannot be Softmax when the number of neurons is 1.\n"; - - throw runtime_error(buffer.str()); - } - - if(neurons_number != 1 && new_activation_function == ActivationFunction::Binary) - { - ostringstream buffer; - - buffer << "OpenNN Exception: ProbabilisticLayer3D class.\n" - << "void set_activation_function(const ActivationFunction&) method.\n" - << "Activation function cannot be Binary when the number of neurons is greater than 1.\n"; - - throw runtime_error(buffer.str()); - } - - if(neurons_number != 1 && new_activation_function == ActivationFunction::Logistic) - { - ostringstream buffer; - - buffer << "OpenNN Exception: ProbabilisticLayer3D class.\n" - << "void set_activation_function(const ActivationFunction&) method.\n" - << "Activation function cannot be Logistic when the number of neurons is greater than 1.\n"; - - throw runtime_error(buffer.str()); - } - -#endif - activation_function = new_activation_function; } @@ -519,7 +467,7 @@ void ProbabilisticLayer3D::set_parameters_glorot() for (Index i = 0; i < synaptic_weights.size(); i++) { - const type random = static_cast(rand() / (RAND_MAX + 1.0)); + const type random = type(rand() / (RAND_MAX + 1.0)); synaptic_weights(i) = minimum + (maximum - minimum) * random; } @@ -595,8 +543,6 @@ void ProbabilisticLayer3D::back_propagate(const Tensor, // Forward propagation - const Index batch_samples_number = forward_propagation->batch_samples_number; - ProbabilisticLayer3DForwardPropagation* probabilistic_layer_3d_forward_propagation = static_cast(forward_propagation); diff --git a/opennn/quasi_newton_method.cpp b/opennn/quasi_newton_method.cpp index 438abe3e6..3ead1bfe8 100644 --- a/opennn/quasi_newton_method.cpp +++ b/opennn/quasi_newton_method.cpp @@ -372,10 +372,6 @@ void QuasiNewtonMethod::calculate_DFP_inverse_hessian(QuasiNewtonMehtodData& opt void QuasiNewtonMethod::calculate_BFGS_inverse_hessian(QuasiNewtonMehtodData& optimization_data) const { - const NeuralNetwork* neural_network = loss_index->get_neural_network(); - - const Index parameters_number = neural_network->get_parameters_number(); - const Tensor& parameters_difference = optimization_data.parameters_difference; const Tensor& gradient_difference = optimization_data.gradient_difference; @@ -389,7 +385,6 @@ void QuasiNewtonMethod::calculate_BFGS_inverse_hessian(QuasiNewtonMehtodData& op Tensor parameters_difference_dot_gradient_difference; Tensor gradient_dot_hessian_dot_gradient; - parameters_difference_dot_gradient_difference.device(*thread_pool_device) = parameters_difference.contract(gradient_difference, AT_B); diff --git a/opennn/recurrent_layer.cpp b/opennn/recurrent_layer.cpp index 06dbc000e..a1bcbe182 100644 --- a/opennn/recurrent_layer.cpp +++ b/opennn/recurrent_layer.cpp @@ -185,7 +185,7 @@ Tensor RecurrentLayer::get_biases(const Tensor& parameters) co Tensor new_biases(biases_number); - new_biases = parameters.slice(Eigen::array({input_weights_number}), Eigen::array({biases_number})); + new_biases = parameters.slice(Eigen::array({input_weights_number}), Eigen::array({biases_number})); Eigen::array two_dim{{1, biases.dimension(1)}}; @@ -205,7 +205,7 @@ Tensor RecurrentLayer::get_input_weights(const Tensor& paramet const Index input_weights_number = get_input_weights_number(); const Tensor new_inputs_weights - = parameters.slice(Eigen::array({0}), Eigen::array({input_weights_number})); + = parameters.slice(Eigen::array({0}), Eigen::array({input_weights_number})); const Eigen::array two_dim{{inputs_number, neurons_number}}; @@ -228,7 +228,7 @@ Tensor RecurrentLayer::get_recurrent_weights(const Tensor& par const Index start_recurrent_weights_number = (parameters_size - recurrent_weights_number); const Tensor new_synaptic_weights - = parameters.slice(Eigen::array({start_recurrent_weights_number}), Eigen::array({recurrent_weights_number})); + = parameters.slice(Eigen::array({start_recurrent_weights_number}), Eigen::array({recurrent_weights_number})); const Eigen::array two_dim{{neurons_number, neurons_number}}; @@ -750,8 +750,6 @@ void RecurrentLayer::back_propagate(const Tensor, 1>& in Tensor& current_inputs = recurrent_layer_forward_propagation->current_inputs; - Tensor& previous_activations = recurrent_layer_forward_propagation->previous_activations; - const Tensor& outputs = recurrent_layer_forward_propagation->outputs; const Tensor& activations_derivatives = recurrent_layer_forward_propagation->activations_derivatives; @@ -789,8 +787,6 @@ void RecurrentLayer::back_propagate(const Tensor, 1>& in Tensor& input_derivatives = recurrent_layer_back_propagation->input_derivatives; - Index input_weights_number = get_input_weights_number(); - const Eigen::array, 1> combinations_weights_indices = { IndexPair(2, 0) }; for(Index sample_index = 0; sample_index < samples_number; sample_index++) @@ -851,11 +847,14 @@ void RecurrentLayer::back_propagate(const Tensor, 1>& in if (sample_index % timesteps != 0) { + /// @todo parallelize + for (Index neuron_index = 0; neuron_index < neurons_number; neuron_index++) { for (Index activation_index = 0; activation_index < neurons_number; activation_index++) { - combinations_recurrent_weights_derivatives(activation_index, neuron_index, neuron_index) += outputs(sample_index - 1, activation_index); + combinations_recurrent_weights_derivatives(activation_index, neuron_index, neuron_index) + += outputs(sample_index - 1, activation_index); } } } diff --git a/opennn/recurrent_layer.h b/opennn/recurrent_layer.h index 50c197f19..cc00550c1 100644 --- a/opennn/recurrent_layer.h +++ b/opennn/recurrent_layer.h @@ -286,10 +286,8 @@ struct RecurrentLayerBackPropagation : LayerBackPropagation set(new_batch_samples_number, new_layer); } - void set(const Index& new_batch_samples_number, Layer* new_layer) final; - void print() const { diff --git a/opennn/scaling_layer_2d.cpp b/opennn/scaling_layer_2d.cpp index e48e6223c..55bde6a66 100644 --- a/opennn/scaling_layer_2d.cpp +++ b/opennn/scaling_layer_2d.cpp @@ -887,7 +887,7 @@ void ScalingLayer2D::calculate_outputs(type* inputs_data, const Tensor } else if(scaler == Scaler::MeanStandardDeviation) { - const type slope = static_cast(1)/descriptives(i).standard_deviation; + const type slope = type(1)/descriptives(i).standard_deviation; const type intercept = -descriptives(i).mean/descriptives(i).standard_deviation; @@ -895,7 +895,7 @@ void ScalingLayer2D::calculate_outputs(type* inputs_data, const Tensor } else if(scaler == Scaler::StandardDeviation) { - column = static_cast(1/descriptives(i).standard_deviation) * inputs.chip(i, 1);/*column/static_cast(descriptives(i).standard_deviation);*/ + column = type(1/descriptives(i).standard_deviation) * inputs.chip(i, 1);/*column/type(descriptives(i).standard_deviation);*/ } else if(scaler == Scaler::Logarithm) { @@ -937,7 +937,7 @@ void ScalingLayer2D::calculate_outputs(type* inputs_data, const Tensor for(Index i = 0; i < input.size(); i++) { - output(i) = -static_cast(1) + static_cast(2*input(i)/255); + output(i) = -type(1) + type(2*input(i)/255); } } else diff --git a/opennn/statistics.cpp b/opennn/statistics.cpp index 949c19e00..58f15e557 100644 --- a/opennn/statistics.cpp +++ b/opennn/statistics.cpp @@ -2460,21 +2460,6 @@ Tensor median(const Tensor& matrix) { const Index raw_variables_number = matrix.dimension(1); -#ifdef OPENNN_DEBUG - - if(rows_number == 0) - { - ostringstream buffer; - - buffer << "OpenNN Exception: Matrix template.\n" - << "Tensor median() const method.\n" - << "Number of rows must be greater than one.\n"; - - throw runtime_error(buffer.str()); - } - -#endif - // median Tensor median(raw_variables_number); @@ -2518,36 +2503,6 @@ Tensor median(const Tensor& matrix) type median(const Tensor& matrix, const Index& raw_variable_index) { - //const Index rows_number = matrix.dimension(0); - -#ifdef OPENNN_DEBUG - - const Index raw_variables_number = matrix.dimension(1); - - if(rows_number == 0) - { - ostringstream buffer; - - buffer << "OpenNN Exception: Matrix template.\n" - << "type median(const Index&) const method.\n" - << "Number of rows must be greater than one.\n"; - - throw runtime_error(buffer.str()); - } - - if(raw_variable_index >= raw_variables_number) - { - ostringstream buffer; - - buffer << "OpenNN Exception: Matrix template.\n" - << "type median(const Index&) const method.\n" - << "Index of raw_variable must be less than number of raw_variables.\n"; - - throw runtime_error(buffer.str()); - } - -#endif - // median type median = type(0); diff --git a/opennn/stochastic_gradient_descent.cpp b/opennn/stochastic_gradient_descent.cpp index 0c6c4e653..141de585b 100644 --- a/opennn/stochastic_gradient_descent.cpp +++ b/opennn/stochastic_gradient_descent.cpp @@ -157,7 +157,7 @@ void StochasticGradientDescent::set_initial_decay(const type& new_decay) { #ifdef OPENNN_DEBUG - if(new_dacay < type(0)) + if(new_decay < type(0)) { ostringstream buffer; diff --git a/opennn/strings_utilities.cpp b/opennn/strings_utilities.cpp index 163af2739..ec40f3483 100644 --- a/opennn/strings_utilities.cpp +++ b/opennn/strings_utilities.cpp @@ -364,7 +364,8 @@ bool is_numeric_string(const string& str) { stod(str, &index); - if(index == str.size() || (str.find("%") != string::npos && index+1 == str.size())) + if(index == str.size() + || (str.find("%") != string::npos && index+1 == str.size())) { return true; } @@ -2100,25 +2101,6 @@ Tensor str_to_input(const string& input_string) } -string output_to_str(const Tensor& flatten_output_data) -{ -/* - const Index alphabet_length = get_alphabet_length(); - - const Index tensor_size = Index(flatten_output_data.size() / alphabet_length); - - Tensor output_data(tensor_size, alphabet_length); - - copy(/*execution::par, - flatten_output_data.data(), - flatten_output_data.data() + tensor_size, output_data.data()); - - return multiple_one_hot_decode(output_data); -*/ - return string(); -} - - /// Calculate the total number of tokens in the documents. Index count(const Tensor, 1>& documents) @@ -4620,7 +4602,7 @@ Tensor TextAnalytics::get_words_presence_percentage(const Tensor TextAnalytics::calculate_combinated_words_frequency(const Tensor, 1>& tokens, const Index& minimum_frequency, const Index& combinations_length) const @@ -4875,12 +4857,12 @@ void load_documents(const string& path) Tensor document_copy(lines_count); Tensor document_target_copy(lines_count); - copy(/*execution::par, + copy(execution::par, document.data(), document.data() + lines_count, document_copy.data()); - copy(/*execution::par, + copy(execution::par, document_target.data(), document_target.data() + lines_count, document_target_copy.data()); @@ -4972,7 +4954,6 @@ string TextAnalytics::generate_word(TextGenerationAlphabet& text_generation_alph string TextAnalytics::generate_phrase(TextGenerationAlphabet& text_generation_alphabet, const string& first_letters, const Index& length) { -/* const Index alphabet_length = text_generation_alphabet.get_alphabet_length(); if(first_letters.length()*alphabet_length != get_inputs_number()) @@ -5111,7 +5092,7 @@ void TextGenerationAlphabet::create_alphabet() alphabet.resize(text_copy.length()); - copy(/*execution::par, + copy(execution::par, text_copy.begin(), text_copy.end(), alphabet.data()); @@ -5263,7 +5244,7 @@ Tensor TextGenerationAlphabet::str_to_input(const string &input_string) Tensor flatten_input_data(1, input_data.size()); - copy(/*execution::par, + copy(execution::par, input_data.data(), input_data.data() + input_data.size(), flatten_input_data.data()); @@ -5271,22 +5252,6 @@ Tensor TextGenerationAlphabet::str_to_input(const string &input_string) return flatten_input_data; } - -string TextGenerationAlphabet::output_to_str(const Tensor&flatten_output_data) const -{ - const Index alphabet_length = get_alphabet_length(); - - const Index tensor_size = Index(flatten_output_data.size()/alphabet_length); - - Tensor output_data(tensor_size, alphabet_length); - - copy(/*execution::par, - flatten_output_data.data(), - flatten_output_data.data() + tensor_size, output_data.data()); - - return multiple_one_hot_decode(output_data); -} - } */ diff --git a/opennn/strings_utilities.h b/opennn/strings_utilities.h index ccc262496..172ca5eca 100644 --- a/opennn/strings_utilities.h +++ b/opennn/strings_utilities.h @@ -133,8 +133,6 @@ namespace opennn Tensor str_to_input(const string&); - string output_to_str(const Tensor&); - // Preprocess methods Index count(const Tensor, 1>& documents); @@ -444,8 +442,6 @@ class TextGenerationAlphabet Tensor str_to_input(const string &) const; - string output_to_str(const Tensor&) const; - private: string text; diff --git a/opennn/sum_squared_error.cpp b/opennn/sum_squared_error.cpp index c55719303..d424939b2 100644 --- a/opennn/sum_squared_error.cpp +++ b/opennn/sum_squared_error.cpp @@ -87,8 +87,6 @@ void SumSquaredError::calculate_output_delta(const Batch&, ForwardPropagation&, BackPropagation& back_propagation) const { - const Index last_trainable_layer_index = neural_network->get_last_trainable_layer_index(); - // Back propagation const Tensor& errors = back_propagation.errors; @@ -113,14 +111,8 @@ void SumSquaredError::calculate_output_delta_lm(const Batch&, #endif - // Neural network - - const Index last_trainable_layer_index = neural_network->get_last_trainable_layer_index(); - // Back propagation - LayerBackPropagationLM* output_layer_back_propagation = back_propagation.neural_network.layers(last_trainable_layer_index); - const Tensor& errors = back_propagation.errors; const Tensor& squared_errors = back_propagation.squared_errors; @@ -155,9 +147,8 @@ void SumSquaredError::calculate_error_gradient_lm(const Batch& , void SumSquaredError::calculate_error_hessian_lm(const Batch&, - BackPropagationLM& back_propagation_lm) const + BackPropagationLM& back_propagation_lm) const { - const Tensor& squared_errors = back_propagation_lm.squared_errors; const Tensor& squared_errors_jacobian = back_propagation_lm.squared_errors_jacobian; Tensor& hessian = back_propagation_lm.hessian; diff --git a/opennn/tensors.h b/opennn/tensors.h index eb53b72aa..9c9b2104e 100644 --- a/opennn/tensors.h +++ b/opennn/tensors.h @@ -65,7 +65,6 @@ void sum_matrices(ThreadPoolDevice*, const Tensor&, Tensor&); void substract_columns(ThreadPoolDevice*, const Tensor&, Tensor&); void substract_matrices(ThreadPoolDevice*, const Tensor&, Tensor&); - void set_identity(Tensor&); void sum_diagonal(Tensor&, const type&); diff --git a/opennn/testing_analysis.cpp b/opennn/testing_analysis.cpp index 6f306811d..c1d20aebe 100644 --- a/opennn/testing_analysis.cpp +++ b/opennn/testing_analysis.cpp @@ -1340,13 +1340,12 @@ Tensor TestingAnalysis::calculate_confusion() const const Index samples_number = targets.dimension(0); const Tensor input_variables_dimensions = data_set->get_input_variables_dimensions(); - if (input_variables_dimensions.size() == 1) { const Tensor outputs = neural_network->calculate_outputs(inputs); - return calculate_confusion_classification(outputs, targets, outputs_number); + return calculate_confusion(outputs, targets, outputs_number); } else if (input_variables_dimensions.size() == 2) { @@ -1362,13 +1361,14 @@ Tensor TestingAnalysis::calculate_confusion() const const Tensor outputs = neural_network->calculate_outputs(inputs_4d); - return calculate_confusion_classification(outputs, targets, outputs_number); + return calculate_confusion(outputs, targets, outputs_number); } + return Tensor(); } -Tensor TestingAnalysis::calculate_confusion_classification(const Tensor& outputs, const Tensor& targets, Index outputs_number) const +Tensor TestingAnalysis::calculate_confusion(const Tensor& outputs, const Tensor& targets, Index outputs_number) const { if (outputs_number == 1) { diff --git a/opennn/testing_analysis.h b/opennn/testing_analysis.h index 8d250c343..f9055d9bf 100644 --- a/opennn/testing_analysis.h +++ b/opennn/testing_analysis.h @@ -226,7 +226,7 @@ class TestingAnalysis Tensor calculate_confusion_binary_classification(const Tensor&, const Tensor&, const type&) const; Tensor calculate_confusion_multiple_classification(const Tensor&, const Tensor&) const; - Tensor calculate_confusion_classification(const Tensor&, const Tensor&, Index) const; + Tensor calculate_confusion(const Tensor&, const Tensor&, Index) const; Tensor calculate_positives_negatives_rate(const Tensor&, const Tensor&) const; diff --git a/opennn/tinyxml2.cpp b/opennn/tinyxml2.cpp index bae6ff18b..286f40f14 100644 --- a/opennn/tinyxml2.cpp +++ b/opennn/tinyxml2.cpp @@ -2650,7 +2650,7 @@ void XMLPrinter::Write(const char* data, int size ) } else { - char* p = _buffer.PushArr(static_cast(size)) - 1; // back up over the null terminator. + char* p = _buffer.PushArr(int(size)) - 1; // back up over the null terminator. memcpy(p, data, size ); p[size] = 0; } diff --git a/opennn/tinyxml2.h b/opennn/tinyxml2.h index 6c291665f..2022e1412 100644 --- a/opennn/tinyxml2.h +++ b/opennn/tinyxml2.h @@ -1683,7 +1683,7 @@ class TINYXML2_LIB XMLDocument : public XMLNode // specified, TinyXML-2 will assume 'xml' points to a // null terminated string. - XMLError Parse(const char* xml, int nBytes= static_cast(-1)); + XMLError Parse(const char* xml, int nBytes= int(-1)); // Load an XML file from disk. @@ -2246,7 +2246,7 @@ class TINYXML2_LIB XMLPrinter : public XMLVisitor virtual void PrintSpace(int depth ); void Print(const char* format, ...); void Write(const char* data, int size ); - inline void Write(const char* data ) { Write(data, static_cast(strlen(data))); } + inline void Write(const char* data ) { Write(data, int(strlen(data))); } void Putc(char ch ); void SealElementIfJustOpened(); diff --git a/opennn/transformer.cpp b/opennn/transformer.cpp index 0c65fa73e..b75950711 100644 --- a/opennn/transformer.cpp +++ b/opennn/transformer.cpp @@ -413,12 +413,12 @@ void Transformer::tokenize_whitespace(const Tensor& context_tokens, T void Transformer::tokenize_wordpiece(const Tensor& context_tokens, Tensor& context) { unordered_map context_vocabulary_map; - for (Index i = 0; i < context_vocabulary.size(); i++) context_vocabulary_map[context_vocabulary(i)] = type(i); - const Index context_vocabulary_size = context_vocabulary.size(); + for (Index i = 0; i < context_vocabulary.size(); i++) + context_vocabulary_map[context_vocabulary(i)] = type(i); - Index token_counter; - bool line_ended; + Index token_counter = 1; + bool line_ended = false; string word; string wordpiece; @@ -427,10 +427,6 @@ void Transformer::tokenize_wordpiece(const Tensor& context_tokens, Te auto wordpiece_entry = context_vocabulary_map.find(""); bool tokenized; - token_counter = 1; - - line_ended = false; - for (Index j = 0; j < context_length - 1; j++) { if (j < context_tokens.size() && token_counter < context_length - 1) @@ -485,7 +481,8 @@ void Transformer::tokenize_wordpiece(const Tensor& context_tokens, Te } else { - if (j == context_tokens.size() || (token_counter == context_length - 1 && !line_ended)) + if (j == context_tokens.size() + || (token_counter == context_length - 1 && !line_ended)) { context(token_counter) = 3; // end indicator token_counter++; diff --git a/opennn/weighted_squared_error.cpp b/opennn/weighted_squared_error.cpp index fe198bbbf..1efa730f3 100644 --- a/opennn/weighted_squared_error.cpp +++ b/opennn/weighted_squared_error.cpp @@ -281,10 +281,6 @@ void WeightedSquaredError::calculate_output_delta(const Batch& batch, const Index total_samples_number = data_set->get_samples_number(); - // Neural network - - const Index last_trainable_layer_index = neural_network->get_last_trainable_layer_index(); - // Batch const Index batch_samples_number = batch.targets_dimensions[0]; @@ -297,8 +293,6 @@ void WeightedSquaredError::calculate_output_delta(const Batch& batch, const Tensor& errors = back_propagation.errors; - LayerBackPropagation* output_layer_back_propagation = back_propagation.neural_network.layers(last_trainable_layer_index); - const pair deltas_pair = back_propagation.get_output_deltas_pair(); TensorMap> deltas(deltas_pair.first, deltas_pair.second[0], deltas_pair.second[1]); @@ -460,11 +454,11 @@ void WeightedSquaredError::from_XML(const tinyxml2::XMLDocument& document) if(negatives_weight_element) { - const string string = negatives_weight_element->GetText(); + const string str = negatives_weight_element->GetText(); try { - set_negatives_weight(type(atof(string.c_str()))); + set_negatives_weight(type(atof(str.c_str()))); } catch(const exception& e) { diff --git a/opennn/weighted_squared_error.h b/opennn/weighted_squared_error.h index 3c0f523f7..52c60493c 100644 --- a/opennn/weighted_squared_error.h +++ b/opennn/weighted_squared_error.h @@ -63,7 +63,7 @@ class WeightedSquaredError : public LossIndex void set_weights(); - void set_normalization_coefficient() override; + void set_normalization_coefficient(); void set_data_set(DataSet*) final;