Skip to content

Commit 404c803

Browse files
SanftMonsterOceania2018
SanftMonster
authored and
Oceania2018
committed
Support the multiple inputs of keras model.fit.
1 parent 7465337 commit 404c803

File tree

21 files changed

+343
-60
lines changed

21 files changed

+343
-60
lines changed

‎src/TensorFlowNET.Core/Data/DatasetV2.cs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ public class DatasetV2 : IDatasetV2
1919

2020
public TensorSpec[] structure { get; set; }
2121

22+
public int FirstInputTensorCount { get; set; } = 1;
23+
2224
public Shape[] output_shapes => structure.Select(x => x.shape).ToArray();
2325

2426
public TF_DataType[] output_types => structure.Select(x => x.dtype).ToArray();
@@ -131,6 +133,7 @@ public IDatasetV2 apply_options()
131133

132134
// (4) Apply stats aggregator options
133135

136+
dataset.FirstInputTensorCount = this.FirstInputTensorCount;
134137
return dataset;
135138
}
136139

@@ -142,7 +145,7 @@ public override string ToString()
142145
$"types: {string.Join(", ", structure.Select(x => "tf." + x.dtype.as_numpy_name()))}, " +
143146
$"len: {length}";
144147

145-
public IEnumerator<(Tensor, Tensor)> GetEnumerator()
148+
public IEnumerator<(Tensors, Tensors)> GetEnumerator()
146149
{
147150
using var ownedIterator = new OwnedIterator(this);
148151

@@ -158,7 +161,8 @@ public override string ToString()
158161
break;
159162
}
160163

161-
yield return (results[0], results.Length == 1 ? null : results[1]);
164+
yield return (new Tensors(results.Take(FirstInputTensorCount)), results.Length == FirstInputTensorCount ?
165+
null : new Tensors(results.Skip(FirstInputTensorCount)));
162166
}
163167
}
164168

‎src/TensorFlowNET.Core/Data/IDatasetV2.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
namespace Tensorflow
66
{
7-
public interface IDatasetV2 : IEnumerable<(Tensor, Tensor)>
7+
public interface IDatasetV2 : IEnumerable<(Tensors, Tensors)>
88
{
99
string[] class_names { get; set; }
1010

@@ -18,6 +18,8 @@ public interface IDatasetV2 : IEnumerable<(Tensor, Tensor)>
1818

1919
TensorSpec[] structure { get; set; }
2020

21+
int FirstInputTensorCount { get; set; }
22+
2123
/// <summary>
2224
/// Caches the elements in this dataset.
2325
/// </summary>

‎src/TensorFlowNET.Core/Data/OwnedIterator.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ void _create_iterator(IDatasetV2 dataset)
2727
_dataset = dataset;
2828
_element_spec = dataset.element_spec;
2929
// _flat_output_types =
30-
(_iterator_resource, _deleter) = ops.anonymous_iterator_v2(_dataset.output_types, _dataset.output_shapes);
30+
_iterator_resource = ops.anonymous_iterator_v3(_dataset.output_types, _dataset.output_shapes);
31+
// TODO(Rinne): deal with graph mode.
3132
ops.make_iterator(dataset.variant_tensor, _iterator_resource);
3233
}
3334

@@ -48,7 +49,7 @@ public Tensor[] next()
4849

4950
public void Dispose()
5051
{
51-
tf.Runner.Execute(tf.Context, "DeleteIterator", 0, new[] { _iterator_resource, _deleter }, null);
52+
//tf.Runner.Execute(tf.Context, "DeleteIterator", 0, new[] { _iterator_resource, _deleter }, null);
5253
}
5354
}
5455
}

‎src/TensorFlowNET.Core/Keras/ArgsDefinition/DataAdapterArgs.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ namespace Tensorflow.Keras.ArgsDefinition
55
{
66
public class DataAdapterArgs: IKerasConfig
77
{
8-
public Tensor X { get; set; }
9-
public Tensor Y { get; set; }
8+
public Tensors X { get; set; }
9+
public Tensors Y { get; set; }
1010
public IDatasetV2 Dataset { get; set; }
1111
public int BatchSize { get; set; } = 32;
1212
public int Steps { get; set; }

‎src/TensorFlowNET.Core/Keras/ArgsDefinition/DataHandlerArgs.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ namespace Tensorflow.Keras.ArgsDefinition
55
{
66
public class DataHandlerArgs: IKerasConfig
77
{
8-
public Tensor X { get; set; }
9-
public Tensor Y { get; set; }
8+
public Tensors X { get; set; }
9+
public Tensors Y { get; set; }
1010
public IDatasetV2 Dataset { get; set; }
1111
public int BatchSize { get; set; } = 32;
1212
public int StepsPerEpoch { get; set; } = -1;

‎src/TensorFlowNET.Core/Keras/Engine/IModel.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,17 @@ ICallback fit(NDArray x, NDArray y,
2424
int workers = 1,
2525
bool use_multiprocessing = false);
2626

27+
ICallback fit(IEnumerable<NDArray> x, NDArray y,
28+
int batch_size = -1,
29+
int epochs = 1,
30+
int verbose = 1,
31+
float validation_split = 0f,
32+
bool shuffle = true,
33+
int initial_epoch = 0,
34+
int max_queue_size = 10,
35+
int workers = 1,
36+
bool use_multiprocessing = false);
37+
2738
void save(string filepath,
2839
bool overwrite = true,
2940
bool include_optimizer = true,

‎src/TensorFlowNET.Core/NumPy/NDArray.Implicit.cs‎

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,76 @@ public void Deconstruct(out byte blue, out byte green, out byte red)
1414
red = data[2];
1515
}
1616

17-
public static implicit operator NDArray(Array array)
17+
public static implicit operator NDArray(int[] array)
18+
=> new NDArray(array);
19+
20+
public static implicit operator NDArray(byte[] array)
21+
=> new NDArray(array);
22+
23+
public static implicit operator NDArray(float[] array)
24+
=> new NDArray(array);
25+
26+
public static implicit operator NDArray(double[] array)
27+
=> new NDArray(array);
28+
29+
public static implicit operator NDArray(long[] array)
30+
=> new NDArray(array);
31+
32+
public static implicit operator NDArray(bool[] array)
33+
=> new NDArray(array);
34+
35+
public static implicit operator NDArray(uint[] array)
36+
=> new NDArray(array);
37+
38+
public static implicit operator NDArray(ulong[] array)
39+
=> new NDArray(array);
40+
41+
public static implicit operator NDArray(int[,] array)
42+
=> new NDArray(array);
43+
44+
public static implicit operator NDArray(byte[,] array)
45+
=> new NDArray(array);
46+
47+
public static implicit operator NDArray(float[,] array)
48+
=> new NDArray(array);
49+
50+
public static implicit operator NDArray(double[,] array)
51+
=> new NDArray(array);
52+
53+
public static implicit operator NDArray(long[,] array)
54+
=> new NDArray(array);
55+
56+
public static implicit operator NDArray(bool[,] array)
57+
=> new NDArray(array);
58+
59+
public static implicit operator NDArray(uint[,] array)
60+
=> new NDArray(array);
61+
62+
public static implicit operator NDArray(ulong[,] array)
63+
=> new NDArray(array);
64+
65+
public static implicit operator NDArray(int[,,] array)
66+
=> new NDArray(array);
67+
68+
public static implicit operator NDArray(byte[,,] array)
69+
=> new NDArray(array);
70+
71+
public static implicit operator NDArray(float[,,] array)
72+
=> new NDArray(array);
73+
74+
public static implicit operator NDArray(double[,,] array)
75+
=> new NDArray(array);
76+
77+
public static implicit operator NDArray(long[,,] array)
78+
=> new NDArray(array);
79+
80+
public static implicit operator NDArray(bool[,,] array)
81+
=> new NDArray(array);
82+
83+
public static implicit operator NDArray(uint[,,] array)
84+
=> new NDArray(array);
85+
86+
public static implicit operator NDArray(ulong[,,] array)
1887
=> new NDArray(array);
1988

2089
public unsafe static implicit operator bool(NDArray nd)

‎src/TensorFlowNET.Core/NumPy/Persistence/NpzDictionaryArray.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ private NDArray OpenEntry(ZipArchiveEntry entry)
2525
return array;
2626

2727
using var s = entry.Open();
28-
return LoadMatrix(s);
28+
return (NDArray)LoadMatrix(s);
2929
}
3030

3131
public Array LoadMatrix(Stream stream)

‎src/TensorFlowNET.Core/Numpy/NDArray.cs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,5 +49,8 @@ public IEnumerator<NDArray> GetEnumerator()
4949

5050
IEnumerator IEnumerable.GetEnumerator()
5151
=> GetEnumerator();
52+
53+
public static explicit operator NDArray(Array array)
54+
=> new NDArray(array);
5255
}
5356
}

‎src/TensorFlowNET.Core/Operations/dataset_ops.cs‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
using System;
2+
using Tensorflow.Contexts;
3+
using Tensorflow.Eager;
24
using Tensorflow.Framework.Models;
35
using Tensorflow.Functions;
6+
using Tensorflow.Operations;
47
using static Tensorflow.Binding;
58

69
namespace Tensorflow
@@ -220,6 +223,37 @@ public Tensor model_dataset(Tensor input_dataset,
220223
return (results[0], results[1]);
221224
}
222225

226+
public Tensor anonymous_iterator_v3(TF_DataType[] output_types, Shape[] output_shapes, string name = null)
227+
{
228+
var ctx = tf.Context;
229+
Dictionary<string, object> attrs = new();
230+
attrs["output_types"] = output_types;
231+
attrs["output_shapes"] = output_shapes;
232+
if (ctx.executing_eagerly())
233+
{
234+
try
235+
{
236+
var result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo("AnonymousIteratorV3", name)
237+
{
238+
attrs = attrs
239+
});
240+
return result[0];
241+
}
242+
catch (Exception)
243+
{
244+
return anonymous_iterator_v3_eager_fallback(output_types, output_shapes, name, ctx);
245+
}
246+
}
247+
return tf.OpDefLib._apply_op_helper("AnonymousIteratorV3", name, attrs).outputs[0];
248+
}
249+
250+
public Tensor anonymous_iterator_v3_eager_fallback(TF_DataType[] output_types, Shape[] output_shapes, string name, Context ctx)
251+
{
252+
object[] attrs = new object[] { output_types, output_shapes };
253+
var result = execute.quick_execute("AnonymousIteratorV3", 1, new Tensor[] { }, attrs, ctx, name);
254+
return result[0];
255+
}
256+
223257
/// <summary>
224258
/// Makes a new iterator from the given `dataset` and stores it in `iterator`.
225259
/// </summary>

0 commit comments

Comments
 (0)