c# - 使用二进制格式写入和读取对象，u-sql

Question

所以我目前正在尝试创建几个U-sql 脚本，其中一个将特定对象写入文件，另一个脚本读取文件并取回对象。

编写U-sql脚本:

@bloomTest =
    EXTRACT date DateTime,
            region string,
            tenantName string,
            fileName string,
            modifyingUsers string,
            firstAccess string,
            lastAccess string
    FROM "SearchLog.tsv"
    USING Extractors.Tsv();

@rs0 =
    SELECT sqlProjsend.Send.send(region) AS Obj
    FROM @bloomTest;


OUTPUT @rs0
TO "/output/bloomtestSend.csv"
USING Outputters.Csv();

编写代码隐藏:

 public static class Send
    {

        public static string send(string s)
        {
            IBloomfilter bloom = new BloomFilter(52);

            bloom.Add("hello");

            byte[] bArr = Serialize(bloom);
            return "|" + BitConverter.ToString(bArr).Replace("-", "") + "|";
        }




        static byte[] Serialize(IBloomfilter bloom)
        {
            MemoryStream stream = new MemoryStream();
            IFormatter formatter = new BinaryFormatter();
            formatter.Serialize(stream, bloom);
            return stream.ToArray();
        }
    }

接收脚本:

@bloomTest =
    EXTRACT bloom string

    FROM "/output/bloomtestSend.csv"
    USING Extractors.Csv();

@rs0 =
    SELECT sqlProjrecv.Recieve.recv(bloom) AS bloomRes
    FROM @bloomTest;


OUTPUT @rs0
TO "/output/bloomtestRecv.csv"
USING Outputters.Csv();

接收代码隐藏:

public static class Recieve
    {   
        public static string recv(string b)
        {

            string[] ss = b.Split('|');

            IBloomfilter bloom = Deserialize(StringToByteArray(ss[1]));

            string[] strings = new string[3];
            strings[0] = "nice";
            strings[1] = "world";
            strings[2] = "hello";

            string retVal = "";
            foreach(string s in strings)
            {

            }
            return retVal;
        }

        public static byte[] StringToByteArray(String hex)
        {
            int NumberChars = hex.Length;
            byte[] bytes = new byte[NumberChars / 2];
            for (int i = 0; i < NumberChars; i += 2)
                bytes[i / 2] = Convert.ToByte(hex.Substring(i, 2), 16);
            return bytes;
        }
        private static IBloomfilter Deserialize(byte[] param)
        {
            using (MemoryStream ms = new MemoryStream())
            {
                ms.Write(param, 0, param.Length);
                IFormatter br = new BinaryFormatter();
                ms.Position = 0;
                return br.Deserialize(ms) as BloomFilter;
            }
        }
    }

我在尝试运行 recieve local 时不断遇到的错误是 could not find collection __codeBehind_g4nuvpoe.vvy 这并没有告诉我太多。我尝试发送的 bloomfilter 对象确实有效。我还尝试在另一个没有 u-sql 脚本的项目中使用完全相同的代码，但它可以完美运行:

class Program
    {
        public static byte[] StringToByteArray(String hex)
        {
            int NumberChars = hex.Length;
            byte[] bytes = new byte[NumberChars / 2];
            for (int i = 0; i < NumberChars; i += 2)
                bytes[i / 2] = Convert.ToByte(hex.Substring(i, 2), 16);
            return bytes;
        }
        private static IBloomfilter Deserialize(byte[] param)
        {
            using (MemoryStream ms = new MemoryStream(param))
            {
                IFormatter br = new BinaryFormatter();
                return br.Deserialize(ms) as BloomFilter;
            }
        }

        static byte[] Serialize(IBloomfilter bloom)
        {
            MemoryStream stream = new MemoryStream();
            IFormatter formatter = new BinaryFormatter();
            formatter.Serialize(stream, bloom);
            return stream.ToArray();
        }


        static void Main(string[] args)
        {

            IBloomfilter bloom = new BloomFilter(51);
            bloom.Add("hello");


            byte[] ser = Serialize(bloom);
            string bloomString = BitConverter.ToString(ser).Replace("-", "");


            ser = StringToByteArray(bloomString);

            bloom = Deserialize(ser);


            string[] k = new string[4];

            k[0] = "hello";
            k[1] = "world";
            k[2] = "not";
            k[3] = "this";

            foreach(string s in k)
            {
                if (bloom.Contains(s))
                {
                    Console.WriteLine(s);
                }

            }

            Console.ReadLine();
        }
     }

此代码按您预期的方式运行，这与您认为的 u-sql 实现在功能上没有什么不同，那么实际脚本是否有问题？

非常欢迎任何想法!

Answer 1

我认为您无法通过这种方法做到这一点。U-SQL 代码将在“容器”上执行，您无权读取和写入文件，除非您使用提取器和输出器。也许它可以在本地工作，但不能在 Azure 上工作。

您可以在此处获取自定义运算符的示例:

https://github.com/MicrosoftDocs/azure-docs/blob/master/articles/data-lake-analytics/data-lake-analytics-u-sql-programmability-guide.md