ST>>извиняюсь, вопрос снят. причина оказалась в другом.
P>Покажи правильно решение. Чисто в академических целях.
Ну было несколько ляпов. Расход памяти был изза того что функция leafs написана не правильно. Так как она написана действительно требуются все списки на всех уровнях. Вот оно и держало.

Сейчас это выглядит вот так:
Добавил функции для получения данных из дерева.

{-# LANGUAGE BangPatterns #-}
module Main where

import System.Random
import Data.List

type Point a = a
type Interval a = (Point a, Point a)
type BelongFunction a = Interval a -> Bool
data OcTree a = Node (Interval a) [Point a] [OcTree a]

main = do
    putStrLn $ "Begin"
    seed <- newStdGen
    let randomPoints = randomlist 5000000 seed
    let tree = chopByLevelNumber 15 . chopEmpty . makeOcTree split2 splitInterval2 (0,1) $ randomPoints
    let lastlevel = leafs tree
    let lastlevelCount = map length lastlevel
    let insphere = selector (inSphere 0.33 0.03) tree
    putStrLn $ "  Last level len : " ++ show ( length lastlevel )
    putStrLn $ "  Last level lengths : " ++ show lastlevelCount
    putStrLn $ "  In Sphere : " ++ show insphere    
    putStrLn $ "  In Sphere len : " ++ show (length insphere)
    putStrLn $ "End"

randomlist :: Int -> StdGen -> [Double]
randomlist n = take n . randoms

leafs :: OcTree a -> [[Point a]]
leafs (Node _ items []) = [items]
leafs (Node _ _ forest) = concat $ map leafs forest

--
-- procedure to create an OcTree
--
makeOcTree splitFunc splitIntervalFunc interval items = mkOcTree0 interval items
  where 
    mkOcTree0 int itms = Node int itms treeList
      where  
        itemsList = splitFunc int itms
        intervalList = splitIntervalFunc int        
        treeList = zipWith mkOcTree0 intervalList itemsList

--
-- procedures to split lists and parameters
--

split2 (x,y) items = split2base items 
  where split2base [] = [[],[]]
        split2base (i:is) | i < c     = [i:s1,s2]
                          | otherwise = [s1,i:s2]
          where [s1,s2] = split2base is
                c = 0.5*(x+y) 
          
splitInterval2 (x,y) = let c = 0.5*(x+y) in [(x,c),(c,y)]

--
-- procedures to chop OcTree
--

chopEmpty :: OcTree a -> OcTree a
chopEmpty (Node int items forest) 
    | null items  = Node int items []
    | otherwise   = Node int items (map chopEmpty forest) 

chopByLevelNumber :: Int -> OcTree a -> OcTree a
chopByLevelNumber 0 (Node int lst _) = Node int lst [] 
chopByLevelNumber n (Node int lst forest) = Node int lst (map (chopByLevelNumber (n-1)) forest)

--
-- procedures to make calls to tree
--

-- Return a Function to determine if Interval intersects with circle. Parameters : Center and radius
inSphere :: RealFloat a => Point a -> a -> BelongFunction a
inSphere pt r (x0,x1) = ((pt+r)>=x0 && (pt+r)<x1) || ((pt-r)>=x0 && (pt-r)<x1)

-- Return leafs' points. Belong function says which leafs we are interested in.
selector :: BelongFunction a -> OcTree a -> [Point a]
selector belong (Node interval items [])
    | belong interval   = items
    | otherwise         = []
selector belong (Node interval _ trees)
    | belong interval   = concat . map (selector belong) $ trees 
    | otherwise         = []