My Blog List

LOVE IS LIFE

#htmlcaption1 #htmlcaption2 Stay Connected

Pages

Tuesday, July 16, 2013

SSIS- script task to export data from database table to .csv file

SSIS- script task to export data from database table to .csv file

User::INPUT_FILE_PATH - input file path to get output file name same as the input file name

User::OutputPath - output path where we get the output in .csv format

User::outputdatabaseConnectionString - Database connection string




/*
   Microsoft SQL Server Integration Services Script Task
   Write scripts using Microsoft Visual C# 2008.
   The ScriptMain is the entry point class of the script.
*/

using System;
using System.Data;
using Microsoft.SqlServer.Dts.Runtime;
using System.Windows.Forms;
using System.Web;
using System.Collections.Generic;

//using System.Linq;
//using System.Web.Security;
//using System.Web.UI;
//using System.Web.UI.WebControls;
//using System.Web.UI.Adapters;
//using System.Web.UI.WebControls.WebParts;
//using System.Web.UI.HtmlControls;
using System.Data.SqlClient;
using System.Text;
using System.IO;


namespace ST_f9e64226a54d4da28a70f42ab090a637.csproj
{
    [System.AddIn.AddIn("ScriptMain", Version = "1.0", Publisher = "", Description = "")]
    public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
    {
        string _FileName = "";
        string unicode = "utf-8";
        #region VSTA generated code
        enum ScriptResults
        {
            Success = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success,
            Failure = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
        };
        #endregion

       
        public void Main()
        {
            // TODO: Add your code here
            string path = Dts.Variables["User::INPUT_FILE_PATH"].Value.ToString();
            String fname = Dts.Variables["User::OutputPath"].Value.ToString() + "\\" + GetFileNames(path);
            Dts.TaskResult = (int)ScriptResults.Success;
            //GetFileNames(path);
            CreateCSVFile(GetTableData(), fname);
        }
        //public static Encoding GetEncoding(string unicode);

        public string GetFileNames(string path)
        {

            string tFileName = new DirectoryInfo(path).GetFiles("*.csv")[0].Name;

            //string[] files = Directory.GetFiles(path);
            //string tfileName=files[0].ToString(); //as assuming always one file
            string[] fileparts = tFileName.Split('_');
            string filePart = fileparts[0] + "_" + fileparts[1] + "_" + fileparts[2] + "_" + "Othertext" + "_" + fileparts[5];


            _FileName = filePart;
            return _FileName;
        }


        public DataTable GetTableData()
        {

            String connstring = Dts.Variables["User::outputdatabaseConnectionString"].Value.ToString();
           
            SqlConnection conn = new SqlConnection(connstring);

            SqlCommand cmd = new SqlCommand("select * from table", conn);
            cmd.CommandType = CommandType.Text;
            SqlDataAdapter adap = new SqlDataAdapter(cmd);
            DataTable dt = new DataTable();
            adap.Fill(dt);
            return dt;
        }


        public void CreateCSVFile(DataTable dt, string strFilePath)
        {
            StreamWriter sw = new StreamWriter(strFilePath, false);
            int iColCount = dt.Columns.Count;
            for (int i = 0; i < iColCount; i++)
            {
                sw.Write(dt.Columns[i]);
                if (i < iColCount - 1)
                {
                    sw.Write(",");
                }
            }
            sw.Write(sw.NewLine);
            // Now write all the rows.
            foreach (DataRow dr in dt.Rows)
            {
                for (int i = 0; i < iColCount; i++)
                {
                    if (!Convert.IsDBNull(dr[i]))
                    {
                        sw.Write(dr[i].ToString());
                    }
                    if (i < iColCount - 1)
                    {
                        sw.Write(",");
                    }
                }
                sw.Write(sw.NewLine);
            }
            sw.Close();
        }

    }

}



7 comments:

  1. I have some weird characters like – . How can I solve code page problem?
    Thanks

    ReplyDelete
    Replies
    1. Hi Arslan,

      Do you have the – characters in the database? Can you send one sample row for me to understand better.

      Delete
    2. If i have understood the problem of yours you can use the below line of code in place of the
      StreamWriter sw = new StreamWriter(strFilePath, false);

      as

      StreamWriter sw = new StreamWriter(new FileStream(strFilePath, FileMode.CreateNew, FileAccess.Write), Encoding.GetEncoding("shift-jis"));

      Here "shift-jis" is the encoding pattern for Japan languages like "病院区分". you search for the encoding language of the data you are receiving for and in place of "shift-jis" use that encoding.

      Delete
  2. Hello Indranil,
    Thank you for your blog, I have searched countless blogs and this is far the easiest example. I was hoping if you could please provide assistance as to where you would put code to cleanup rogue CR LF within the double quotes? Any help appreciated and thank you again

    ReplyDelete
  3. Thanks.
    Am getting " An exception of type 'System.Data.SqlClient.SqlException' occurred in System.Data.dll but was not handled in user code " error for bigger table.
    Also i need to handle null value ( if NULL then NULL and id '' then '')
    Can you pls help us for above issue

    ReplyDelete


  4. Hi Thanks for the good representation of the export to CSV... but I tried this and its working well for less data, I have to export some 15 tables which has around 10 Million records each... The package fails throwing some exception from the Script task... is there something been missed for handling this... find below the exception that is received from the script task


    System.OutOfMemoryException: Exception of type 'System.OutOfMemoryException' was thrown.
    at System.Data.RBTree`1.TreePage..ctor(Int32 size)
    at System.Data.RBTree`1.AllocPage(Int32 size)
    at System.Data.RBTree`1.GetNewNode(K key)
    at System.Data.Index.InitRecords(IFilter filter)
    at System.Data.Index..ctor(DataTable table, IndexField[] indexFields, Comparison`1 comparison, DataViewRowState recordStates, IFilter rowFilter)
    at System.Data.DataTable.GetIndex(IndexField[] indexDesc, DataViewRowState recordStates, IFilter rowFilter)
    at System.Data.DataColumn.get_SortIndex()
    at System.Data.DataColumn.IsNotAllowDBNullViolated()
    at System.Data.DataTable.EnableConstraints()
    at System.Data.DataTable.set_EnforceConstraints(Boolean value)
    at System.Data.DataTable.EndLoadData()
    at System.Data.Common.DataAdapter.FillFromReader(DataSet dataset, DataTable datatable, String srcTable, DataReaderContainer dataReader, Int32 startRecord, Int32 maxRecords, DataColumn parentChapterColumn, Object parentChapterValue)
    at System.Data.Common.DataAdapter.Fill(DataTable[] dataTables, IDataReader dataReader, Int32 startRecord, Int32 maxRecords)
    at System.Data.Common.LoadAdapter.FillFromReader(DataTable[] dataTables, IDataReader dataReader, Int32 startRecord, Int32 maxRecords)
    at System.Data.DataTable.Load(IDataReader reader, LoadOption loadOption, FillErrorEventHandler errorHandler)
    at ST_e687e423204547fc9889e990c3f593da.ScriptMain.Main()

    ReplyDelete
    Replies
    1. Were you able to fix this "OutOfMemmoryException". I have similar issue while exporting millions of records. Any help Appreciated. thank you.

      Delete