BenchmarkDotNet=v0.10.14, OS=Windows 10.0.15063.1155 (1703/CreatorsUpdate/Redstone2)
Intel Core i7-6600U CPU 2.60GHz (Skylake), 1 CPU, 4 logical and 2 physical cores
Frequency=2742189 Hz, Resolution=364.6722 ns, Timer=TSC
  [Host]       : .NET Framework 4.6.1 (CLR 4.0.30319.42000), 64bit RyuJIT-v4.7.2671.0
  LegacyJitX86 : .NET Framework 4.6.1 (CLR 4.0.30319.42000), 32bit LegacyJIT-v4.7.2671.0

Job=LegacyJitX86  Jit=LegacyJit  Platform=X86
Runtime=Clr

     Method |     Mean |     Error |    StdDev | Scaled |    Gen 0 |    Gen 1 |    Gen 2 | Allocated |
----------- |---------:|----------:|----------:|-------:|---------:|---------:|---------:|----------:|
 UnsafeLinq | 4.256 ms | 0.0849 ms | 0.0978 ms |   0.43 | 570.3125 | 375.0000 | 375.0000 |   3.83 MB |
    Perfect | 9.989 ms | 0.1198 ms | 0.1120 ms |   1.00 | 156.2500 | 156.2500 | 156.2500 |   3.81 MB |

public int[,] UnsafeLinq() => from d in data select (d[-1, 0] + d[1, 0] + d[0, -1] + d[0, 1]) / 4;

        public static Filter<int, int> GenerateUnsafeFilter(Kernel<int, int> kernel)
        {
            var km = KernelMeasure.Measure(kernel);

            var ab = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName("FixedTest"), AssemblyBuilderAccess.RunAndCollect);
            var tb = ab.DefineDynamicModule("FixedTest", "FixedTest.dll").DefineType("Filter." + kernel.Method.Name, TypeAttributes.Class | TypeAttributes.Public);
            var dm = tb.DefineMethod("Filter", MethodAttributes.HideBySig | MethodAttributes.Public | MethodAttributes.Static, typeof(int[,]), new Type[] { typeof(object), typeof(int[,]) });

            var ilg = dm.GetILGenerator();

            var target = ilg.DeclareLocal(typeof(int[,]), true);
            var source = ilg.DeclareLocal(typeof(int[,]), true); // a replica of the arg1, to force the GC pinning
            var psource = ilg.DeclareLocal(typeof(int*));
            var ptarget = ilg.DeclareLocal(typeof(int*));
            var i_var = ilg.DeclareLocal(typeof(int));
            var j_var = ilg.DeclareLocal(typeof(int));
            var h_var = ilg.DeclareLocal(typeof(int));
            var w_var = ilg.DeclareLocal(typeof(int));
            var W_var = ilg.DeclareLocal(typeof(int));

            ilg.Emit(OpCodes.Ldarg_1); // source
            ilg.Emit(OpCodes.Ldc_I4_0);// 0
            ilg.Emit(OpCodes.Call, typeof(int[,]).GetMethod("GetLength", new Type[] { typeof(int) }));
            ilg.Emit(OpCodes.Stloc, h_var);


            ilg.Emit(OpCodes.Ldarg_1); // source
            ilg.Emit(OpCodes.Ldc_I4_1);// 0
            ilg.Emit(OpCodes.Call, typeof(int[,]).GetMethod("GetLength", new Type[] { typeof(int) }));
            ilg.Emit(OpCodes.Stloc, w_var);

            ilg.Emit(OpCodes.Ldloc, h_var);
            ilg.Emit(OpCodes.Ldloc, w_var);
            ilg.Emit(OpCodes.Newobj, typeof(int[,]).GetConstructor(new Type[] { typeof(int), typeof(int) }));
            ilg.Emit(OpCodes.Stloc, target);

            ilg.Emit(OpCodes.Ldloc, w_var);
            ilg.Emit(OpCodes.Stloc, W_var);

            ilg.EmitIncrease(h_var, -km.ymax);
            ilg.EmitIncrease(w_var, -km.xmax);


            ilg.Emit(OpCodes.Ldarg_1);
            ilg.Emit(OpCodes.Stloc_S, source); // pinning the source array

            ilg.EmitFor(i_var, 0 - km.xmin, h_var, () =>
            {

                ilg.Emit(OpCodes.Ldloc, source);
                ilg.Emit(OpCodes.Ldloc, i_var);
                ilg.Emit(OpCodes.Ldc_I4, -km.ymin);
                ilg.Emit(OpCodes.Call, _arrayAddressGetter);
                ilg.Emit(OpCodes.Conv_U);
                ilg.Emit(OpCodes.Stloc, psource);  // psource = &source[i, lb];

                ilg.Emit(OpCodes.Ldloc, target);
                ilg.Emit(OpCodes.Ldloc, i_var);
                ilg.Emit(OpCodes.Ldc_I4, -km.ymin);
                ilg.Emit(OpCodes.Call, _arrayAddressGetter);
                ilg.Emit(OpCodes.Conv_U);
                ilg.Emit(OpCodes.Stloc, ptarget); // ptarget = &target[i, lb]

                ilg.EmitFor(j_var, 0 - km.ymin, w_var, () =>
                {
                    ilg.Emit(OpCodes.Ldloc_S, ptarget); // :ptarget

                    var usilv = new UnsafeInliningILInstructionVisitor(ilg, i_var, j_var, psource, W_var);
                    new ILReader(kernel.Method).Accept(usilv);
                    //EmitDebug<int>(ilg);

                    ilg.Emit(OpCodes.Stind_I4);

                    ilg.EmitIncrease(psource, 4); // psource++
                    ilg.EmitIncrease(ptarget, 4); // ptarget++
                });
            });

            ilg.Emit(OpCodes.Ldloc, target);
            ilg.Emit(OpCodes.Ret);

            var type = tb.CreateType();
            var mi = type.GetMethod("Filter");
            var inlined = (Func<object, int[,], int[,]>)mi.CreateDelegate(typeof(Func<object, int[,], int[,]>));

            int[,] filter(int[,] data) => inlined(kernel.Target, data);
            return filter;
        }

            protected override void EmitSourceAccessCode()
            {
                //          Opcode                   Stack state (head left)
                //                                     dy, dx, psource, ...
                Target.Emit(OpCodes.Ldloc, _w);    // w, dy, dx, psource
                Target.Emit(OpCodes.Mul);           // w*dy, dx, psource, ...
                Target.Emit(OpCodes.Add);           // w*dy+dx, psource, ...
                Target.Emit(OpCodes.Ldc_I4_4);      // 4, dy + w*dx, psource, ...
                Target.Emit(OpCodes.Mul);           // 4 * (w*dy +  dx), psource, ...
                Target.Emit(OpCodes.Add);           // psource + 4 * dy + 4*w*dx, ...
                Target.Emit(OpCodes.Ldind_I4);      // psource[dy*w+dx]
            }

from d in data select (d[-1, 0] + d[1, 0] + d[0, -1] + d[0, 1]) / 4;

var data = new int[3000, 4000];
// init data with the content
var filtered = from d in data select (d[-1, 0] + d[1, 0] + d[0, -1] + d[0, 1]) / 4; // C4-filtering

using System;
using System.Diagnostics;

namespace ConsoleApp1
{
    interface IArray2dD<T>
    {
        T this[int dx, int dy] { get; set; }

        int dx { get; }
        int dy { get; }
    }

    struct Array2D<T> : IArray2dD<T>
    {
        private readonly T[,] _array;

        public Array2D(int dx, int dy) { _array = new T[dx, dy]; }

        public T this[int x, int y]
        {
            get => _array[x, y];
            set => _array[x, y] = value;
        }

        public int dx { get { return _array.GetLength(0); } }
        public int dy { get { return _array.GetLength(1); } }
    }

    struct Array2DEmu<T> : IArray2dD<T>
    {
        private readonly T[] _array;
        public int dx { get; }
        public int dy { get; }

        public Array2DEmu(int dx, int dy)
        {
            this.dx = dx;
            this.dy = dy;
            _array = new T[dx * dy];
        }

        public T this[int x, int y]
        {
            get => _array[dx * y + x];
            set => _array[dx * y + x] = value;
        }
    }

    interface IFactory<T>
    {
        T create();
    }

    class Program
    {
        static void Test<T, TFactory>()
            where T : struct, IArray2dD<int>
            where TFactory : struct, IFactory<T>
        {
            T array = default(TFactory).create();
            int sum = 0;

            var rnd = new Random((int)DateTime.Now.TimeOfDay.Ticks);

            for (int x = 0; x < array.dx; x++)
                for (int y = 0; y < array.dy; y++)
                    array[x, y] = rnd.Next();

            Stopwatch sw = new Stopwatch();
            sw.Start();

            for (int i = 0; i < 1000; i++)
            {
                for (int x = 0, dx = array.dx; x < dx; x++)
                    for (int y = 0, dy= array.dy; y < dy; y++)
                        sum += array[x, y];
            }

            sw.Stop();

            Console.WriteLine("Elapsed(ms)={0}", sw.Elapsed);
        }

        struct NativeArrayFactory<T> : IFactory<Array2D<T>>
        {
            public Array2D<T> create()
            {
                return new Array2D<T>(1000, 1000);
            }
        }

        struct ArrayEmuFactory<T> : IFactory<Array2DEmu<T>>
        {
            public Array2DEmu<T> create()
            {
                return new Array2DEmu<T>(1000, 1000);
            }
        }

        static void Main(string[] args)
        {
            Console.Write("Native 2D array: ");
            Test<Array2D<int>, NativeArrayFactory<int>>();

            Console.Write("Emulated 2D array: ");
            Test<Array2DEmu<int>, ArrayEmuFactory<int>>();
        }
    }
}

Native 2D array: Elapsed(ms)=00:00:00.7817720
Emulated 2D array: Elapsed(ms)=00:00:00.6798358

    unsafe readonly ref struct SpanIndexer2D<T>
    {
        public Span<T> array { get; }
        public int dx { get; }
        public int dy { get; }

        public SpanIndexer2D(int dx, int dy)
        {
            this.dx = dx;
            this.dy = dy;
            array = new T[dx * dy];
        }

        public T this[int x, int y]
        {
            get => array[dx * y + x];
            set => array[dx * y + x] = value;
        }
    }

    unsafe readonly ref struct UnsafeIntArray2D
    {
        private readonly int* _array;
        public int dx { get; }
        public int dy { get; }

        public UnsafeIntArray2D(int* array, int dx, int dy)
        {
            this.dx = dx;
            this.dy = dy;
            _array = array;
        }

        public int this[int x, int y]
        {
            get => _array[dx * y + x];
            set => _array[dx * y + x] = value;
        }
    }

//...

        static void SpanTest()
        {
            int dx = 1000, dy = 1000;
            var array = new SpanIndexer2D<int>(dx, dy);

            int sum = 0;

            var rnd = new Random((int)DateTime.Now.TimeOfDay.Ticks);

            for (int x = 0; x < array.dx; x++)
                for (int y = 0; y < array.dy; y++)
                    array[x, y] = rnd.Next();

            Stopwatch sw = new Stopwatch();
            sw.Start();

            for (int i = 0; i < 1000; i++)
            {
                for (int x = 0; x < dx; x++)
                    for (int y = 0; y < dy; y++)
                        sum += array[x, y];
            }

            sw.Stop();

            Console.WriteLine("Elapsed(ms)={0}", sw.Elapsed);
        }

        static unsafe void UnsafeTest()
        {
            int dx = 1000, dy = 1000;
            var indexer = new SpanIndexer2D<int>(dx, dy);

            fixed (int* intArray = indexer.array)
            {
                var array = new UnsafeIntArray2D(intArray, dx, dy);

                int sum = 0;

                var rnd = new Random((int)DateTime.Now.TimeOfDay.Ticks);

                for (int x = 0; x < array.dx; x++)
                    for (int y = 0; y < array.dy; y++)
                        array[x, y] = rnd.Next();

                Stopwatch sw = new Stopwatch();
                sw.Start();

                for (int i = 0; i < 1000; i++)
                {
                    for (int x = 0; x < dx; x++)
                        for (int y = 0; y < dy; y++)
                            sum += array[x, y];
                }

                sw.Stop();

                Console.WriteLine("Elapsed(ms)={0}", sw.Elapsed);
            }
        }

Native 2D array: Elapsed(ms)=00:00:00.7712649
Emulated 2D array: Elapsed(ms)=00:00:00.6893393
Span 2D array: Elapsed(ms)=00:00:00.9882120
Unsafe 2D array: Elapsed(ms)=00:00:00.5858264