học Wide & Sâu cho lỗi dữ liệu lớn: GraphDef không thể lớn hơn 2GB

Question

Chèn 1mm + hàng vào wide and deep learning model ném ValueError: GraphDef cannot be larger than 2GB:

Traceback (most recent call last): 
    File "search_click.py", line 207, in <module> 
    tf.app.run() 
    File "/usr/lib/python2.7/site-packages/tensorflow/python/platform/app.py", line 30, in run 
    sys.exit(main(sys.argv)) 
    File "search_click.py", line 204, in main 
    train_and_eval() 
    File "search_click.py", line 181, in train_and_eval 
    m.fit(input_fn=lambda: input_fn(df_train), steps=FLAGS.train_steps) 
    File "/usr/lib/python2.7/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 182, in fit 
    monitors=monitors) 
    File "/usr/lib/python2.7/site-packages/tensorflow/contrib/learn/python/learn/estimators/estimator.py", line 458, in _train_model 
    summary_writer=graph_actions.get_summary_writer(self._model_dir)) 
    File "/usr/lib/python2.7/site-packages/tensorflow/contrib/learn/python/learn/graph_actions.py", line 76, in get_summary_writer 
    graph=ops.get_default_graph()) 
    File "/usr/lib/python2.7/site-packages/tensorflow/python/training/summary_io.py", line 113, in __init__ 
    self.add_graph(graph=graph, graph_def=graph_def) 
    File "/usr/lib/python2.7/site-packages/tensorflow/python/training/summary_io.py", line 204, in add_graph 
    true_graph_def = graph.as_graph_def(add_shapes=True) 
    File "/usr/lib/python2.7/site-packages/tensorflow/python/framework/ops.py", line 2117, in as_graph_def 
    raise ValueError("GraphDef cannot be larger than 2GB.") 
ValueError: GraphDef cannot be larger than 2GB. 

tôi xác định cùng input_fn như trong ví dụ:

def input_fn(df): 
    """Input builder function.""" 
    # Creates a dictionary mapping from each continuous feature column name (k) to 
    # the values of that column stored in a constant Tensor. 
    continuous_cols = {k: tf.constant(df[k].values) for k in CONTINUOUS_COLUMNS} 
    # Creates a dictionary mapping from each categorical feature column name (k) 
    # to the values of that column stored in a tf.SparseTensor. 
    categorical_cols = {k: tf.SparseTensor(
     indices=[[i, 0] for i in range(df[k].size)], 
     values=df[k].values, 
     shape=[df[k].size, 1]) 
         for k in CATEGORICAL_COLUMNS} 
    # Merges the two dictionaries into one. 
    feature_cols = dict(continuous_cols) 
    feature_cols.update(categorical_cols) 
    # Converts the label column into a constant Tensor. 
    label = tf.constant(df[LABEL_COLUMN].values) 
    # Returns the feature columns and the label. 
    return feature_cols, label 

Có thay thế cho tf.constant và tf.SparseTensor cho phép chèn theo lô và tránh lỗi bộ nhớ không?

Answer 1

Ví dụ về nhu cầu rộng và sâu tải toàn bộ dữ liệu vào bộ nhớ. Nếu bạn có tập dữ liệu lớn, bạn có thể cần tf.decode_csv để nhập csv được định dạng. Nếu định dạng đầu vào của bạn tự xác định, bạn nên tạo một custom data reader.

Answer 2

Một solution được tạo bởi @ilblackdragon là sử dụng dataframe.queues. Thật không may, cho các biến phân loại, bạn phải để nguyên mã hóa chúng đầu tiên nếu không bạn nhận được lỗi:

ValueError: Data types for extracting pandas data must be int, float, or bool. Found: 'sex' type='object', 'embarked' type='object' 

Cũng sử dụng một DNNClassifier không hoạt động và cung cấp cho một lỗi quan trọng trên các loại mã hóa (ví dụ KeyError: 'Embarked_ids'). Vì vậy, trong bài viết, tác giả tạo ra mô hình phân loại của riêng mình.

Ví dụ mã:

# -*- coding: utf-8 -*- 
# flake8: noqa ignore=E501 
import tempfile 

import pandas as pd 
import tensorflow as tf 
import tensorflow.contrib.learn as tf_learn 
import tensorflow.contrib.layers as tf_layers 
from sklearn.metrics import accuracy_score 
from sklearn.preprocessing import LabelEncoder 
from sklearn.model_selection import train_test_split 

# Define the column names for the data sets. 
LABEL_COLUMN = 'Survived' 
CONTINUOUS_COLUMNS = ['Age', 'SibSp', 'Parch', 'Fare'] 
CATEGORICAL_COLUMNS = ['Pclass', 'Sex', 'Embarked'] 
CATEGORICAL_ID_COLUMNS = [col + '_ids' for col in CATEGORICAL_COLUMNS] 
FEATURE_COLUMNS = CONTINUOUS_COLUMNS + CATEGORICAL_ID_COLUMNS 

HIDDEN_UNITS = [10, 20, 10] 
CATEGORICAL_EMBED_SIZE = 10 

LABEL_ENCODERS = {} 

def get_feature_cols(): 
    # used in DNNClassifier which doesn't work 
    continuous_features = [tf_layers.real_valued_column(col) for col in 
          CONTINUOUS_COLUMNS] 
    categorical_features = [ 
     tf_layers.embedding_column(
      tf_layers.sparse_column_with_integerized_feature(col + '_ids', len(
       LABEL_ENCODERS[col].classes_)), 
      CATEGORICAL_EMBED_SIZE) 
     for col in CATEGORICAL_COLUMNS 
     ] 
    return continuous_features + categorical_features 


def pandas_input_fn(X, y=None, batch_size=128, num_epochs=None): 
    def input_fn(): 
     if y is not None: 
      X['target'] = y 
     queue = tf_learn.dataframe.queues.feeding_functions.enqueue_data(
      X, 1000, shuffle=num_epochs is None, num_epochs=num_epochs) 
     if num_epochs is None: 
      features = queue.dequeue_many(batch_size) 
     else: 
      features = queue.dequeue_up_to(batch_size) 

     features = dict(zip(['index'] + list(X.columns), features)) 

     if y is not None: 
      target = features.pop('target') 
      return features, target 
     return features 

    return input_fn 


def encode_categorical(df): 
    global LABEL_ENCODERS 
    for col in CATEGORICAL_COLUMNS: 
     if df[col].dtype == 'object': 
      df[col] = df[col].astype(str) 
     encoder = LabelEncoder().fit(df[col]) 
     df[col + '_ids'] = encoder.transform(df[col]) 
     df.pop(col) 
     LABEL_ENCODERS[col] = encoder 
    return df, LABEL_ENCODERS 


def dnn_tanh(features, target, hidden_units=HIDDEN_UNITS): 
    global LABEL_ENCODERS 
    target = tf.one_hot(target, 2, 1.0, 0.0) 

    # Organize continuous features. 
    final_features = [tf.expand_dims(tf.cast(features[col], tf.float32), 1) for 
         col in CONTINUOUS_COLUMNS] 

    # Embed categorical variables into distributed representation. 
    for col in CATEGORICAL_COLUMNS: 
     feature = tf_learn.ops.categorical_variable(
      features[col + '_ids'], 
      len(LABEL_ENCODERS[col].classes_), 
      embedding_size=CATEGORICAL_EMBED_SIZE, 
      name=col) 
     final_features.append(feature) 

    # Concatenate all features into one vector. 
    features = tf.concat(1, final_features) 

    # Deep Neural Network 
    logits = tf_layers.stack(features, 
          tf_layers.fully_connected, 
          stack_args=hidden_units, 
          activation_fn=tf.tanh) 
    prediction, loss = tf_learn.models.logistic_regression(logits, target) 
    train_op = tf_layers.optimize_loss(loss, 
             tf.contrib.framework.get_global_step(), 
             optimizer='SGD', 
             learning_rate=0.05) 
    return tf.argmax(prediction, dimension=1), loss, train_op 


def process_input_df(df): 
    df, label_encoders = encode_categorical(df) 
    y = df.pop(LABEL_COLUMN) 
    X = df[CATEGORICAL_ID_COLUMNS + CONTINUOUS_COLUMNS].fillna(0) 
    return X, y 


def train(X, y, steps=100): 
    model_dir = tempfile.mkdtemp() 
    print("model dir: ", model_dir) 
    classifier = tf_learn.Estimator(model_fn=dnn_tanh, model_dir=model_dir) 
    classifier.fit(input_fn=pandas_input_fn(X, y), steps=steps) 

    ''' 
    # Using DNNClassifier gives KeyError (e.g on EmbedIds) 
    classifier = learn.DNNClassifier(hidden_units=[10, 20, 10], 
     n_classes=2, 
     feature_columns=get_feature_cols(), 
     optimizer=tf.train.GradientDescentOptimizer(learning_rate=0.05)) 
    classifier.fit(X, y, batch_size=128, steps=500) 
    ''' 

    return classifier 


def predict(classifier, X): 
    return list(classifier.predict(input_fn=pandas_input_fn(X, num_epochs=1), 
            as_iterable=True)) 


def evaluate(classifier, X, y, steps=1): 
    results = classifier.evaluate(input_fn=pandas_input_fn(X[FEATURE_COLUMNS], y), 
            steps=steps) 
    for key in sorted(results): 
     print("%s: %s" % (key, results[key])) 


if __name__ == '__main__': 
    # DOWNLOAD TITANIC TRAIN DATA 
    data = pd.read_csv('~/titanic_train.csv') # LOAD DATA 
    X, y = process_input_df(data) 
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0) 
    classifier = train(X_train, y_train, steps=100) 
    print("accuracy_score", accuracy_score(y_test, predict(classifier, X_test))) 
    evaluate(classifier, X_test, y_test, steps=1) 

Answer 3

tôi giải quyết vấn đề này, cuối cùng, sử dụng đoạn mã sau đó được lấy cảm hứng từ những người khác. Hy vọng mã này sẽ hữu ích cho người khác. Cảm ơn tất cả các bạn đã bình luận ở trên.

def input_fn(batch_size,filename): 
    examples_op = tf.contrib.learn.read_batch_examples(
     filename, 
     batch_size=batch_size, 
     reader=tf.TextLineReader, 
     num_epochs=1, 
     parse_fn=lambda x: tf.decode_csv(x, [tf.constant([''], dtype=tf.string)] * len(HEADERS))) 

    examples_dict = {} 
    for i, header in enumerate(HEADERS): 
     examples_dict[header] = examples_op[:, i] 

    feature_cols = {k: tf.string_to_number(examples_dict[k], out_type=tf.float32) 
        for k in CONTINUOUS_COLUMNS} 

    feature_cols.update({k: dense_to_sparse(examples_dict[k]) 
         for k in CATEGORICAL_COLUMNS}) 

    label = tf.string_to_number(examples_dict[LABEL_COLUMN], out_type=tf.int32) 

    return feature_cols, label 

def input_fn_pre(batch_size,filename): 
    examples_op = tf.contrib.learn.read_batch_examples(
     filename, 
     batch_size=batch_size, 
     reader=tf.TextLineReader, 
     num_epochs=1, 
     parse_fn=lambda x: tf.decode_csv(x, [tf.constant([''], dtype=tf.string)] * len(HEADERS))) 

    examples_dict = {} 
    for i, header in enumerate(HEADERS): 
     examples_dict[header] = examples_op[:, i] 

    feature_cols = {k: tf.string_to_number(examples_dict[k], out_type=tf.float32) 
        for k in CONTINUOUS_COLUMNS} 

    feature_cols.update({k: dense_to_sparse(examples_dict[k]) 
         for k in CATEGORICAL_COLUMNS}) 
    return feature_cols 
def dense_to_sparse(dense_tensor): 
    indices = tf.to_int64(tf.transpose([tf.range(tf.shape(dense_tensor)[0]), tf.zeros_like(dense_tensor, dtype=tf.int32)])) 
    values = dense_tensor 
    shape = tf.to_int64([tf.shape(dense_tensor)[0], tf.constant(1)]) 

    return tf.SparseTensor(
     indices=indices, 
     values=values, 
     shape=shape 
    ) 

def train_and_eval(): 
    """Train and evaluate the model.""" 
    data = pd.read_csv('spark_traindata_forrun_no_nanindex.csv',skipinitialspace=True, 
     engine="python") 
    value_range={} 
    for column in CATEGORICAL_COLUMNS: 
    data[column]=data[column].astype(str) 
    value_range[column]=list(set(data[column])) 
    model_dir = './model6' 
    print("model directory = %s" % model_dir) 
    test = pd.read_csv('test.csv',names= HEADERS) 
    m = build_estimator(model_dir,value_range) 
    m.fit(input_fn=lambda: input_fn(128,'train.csv'), steps=FLAGS.train_steps) 
    results = m.evaluate(input_fn=lambda: input_fn(5000,'test.csv'), steps=1) 
    for key in sorted(results): 
    print("%s: %s" % (key, results[key]))